diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 000000000..14920021a --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1 @@ +codecov: {} diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index edc2b6232..b0fba7889 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,10 +1,10 @@ name: Run Black -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: psf/black@26.5.1 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bb65c1465..040d5eb81 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0d2f6e3c9..b0ede9e7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,7 +1,11 @@ name: Run tests -on: [push, pull_request] - +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + id-token: write + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -10,16 +14,18 @@ jobs: test: name: Test - ${{ matrix.python-version }} - ${{matrix.os}} runs-on: ${{matrix.os}} + env: + CODECOV_GPG_FINGERPRINT: 27034E7FDB850E0BBC2C62FF806BB28AED779869 strategy: matrix: - python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15.0-rc.1'] + python-version: ['3.13', '3.14', '3.15.0-rc.1'] os: [ubuntu-latest, windows-latest, macos-latest] fail-fast: false steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -29,9 +35,31 @@ jobs: python -m pip install -e .[dev] - name: Test with pytest run: | - python -m pytest --cov -v + python -m pytest --cov --cov-report=term --cov-report=xml:coverage.xml -v + - name: Authenticate Codecov signing key + shell: bash + run: | + set -euo pipefail + GNUPGHOME="$(python -c 'import os, tempfile; from pathlib import Path; print(Path(tempfile.mkdtemp(prefix="codecov-gnupg-", dir=os.environ["RUNNER_TEMP"])).as_posix())')" + GNUPGHOME="$(cd "$GNUPGHOME" && pwd -P)" + export GNUPGHOME + printf 'GNUPGHOME=%s\n' "$GNUPGHOME" >> "$GITHUB_ENV" + key="$GNUPGHOME/codecov.asc" + curl --fail --location --output "$key" https://keybase.io/codecovsecops/pgp_keys.asc + fingerprint="$( + gpg --batch --with-colons --show-keys "$key" | + awk -F: '$1=="pub"{primary=1;next} primary && $1=="fpr"{print $10;primary=0}' + )" + test "$fingerprint" = "$CODECOV_GPG_FINGERPRINT" + gpg --batch --no-default-keyring --keyring codecov.gpg --import "$key" + printf '%s:6:\n' "$fingerprint" | gpg --batch --import-ownertrust + printf 'no-default-keyring\nkeyring codecov.gpg\nimport-options merge-only\n' > "$GNUPGHOME/gpg.conf" - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: python-rope/rope + use_oidc: true + slug: ${{ github.repository }} + files: coverage.xml + disable_search: true + plugins: noop + fail_ci_if_error: true diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 6a9b4c2cd..278215d8c 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -3,19 +3,66 @@ name: Upload Python Package on: release: types: [published] + workflow_dispatch: + inputs: + release_tag: + description: Existing GitHub Release tag to validate without republishing + required: true + type: string + release_commit: + description: Published release's recorded full tag-target commit SHA + required: true + type: string jobs: + verify-fork-release: + if: github.event_name == 'workflow_dispatch' || contains(github.event.release.tag_name, '+') + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_commit || github.sha }} + - name: Verify published tag still identifies the pinned source + env: + RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }} + RELEASE_COMMIT: ${{ github.event_name == 'workflow_dispatch' && inputs.release_commit || github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$RELEASE_COMMIT" + git fetch --no-tags origin "refs/tags/$RELEASE_TAG" + test "$(git rev-parse 'FETCH_HEAD^{commit}')" = "$(git rev-parse HEAD)" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.13' + - name: Set up native artifact validation + run: | + python -m pip install uv + make setup + - name: Download published fork artifacts + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }} + run: >- + gh release download "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" + --pattern '*.whl' --pattern '*.tar.gz' --dir dist + - name: Validate published wheel and source distribution + run: | + sha256sum dist/*.whl dist/*.tar.gz + make artifact-check + deploy: + if: ${{ github.event_name == 'release' && !contains(github.event.release.tag_name, '+') }} runs-on: ubuntu-latest environment: publishing permissions: id-token: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v7 + uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.14' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/task-completed-checker-action.yml b/.github/workflows/task-completed-checker-action.yml index dc02c4100..d4911155a 100644 --- a/.github/workflows/task-completed-checker-action.yml +++ b/.github/workflows/task-completed-checker-action.yml @@ -6,7 +6,9 @@ on: jobs: task-check: runs-on: ubuntu-latest + permissions: + checks: write steps: - - uses: kentaro-m/task-completed-checker-action@v0.1.0 + - uses: kentaro-m/task-completed-checker-action@15aa21278f36c35b1006de5c65efa6b04e7b02bf # v0.1.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 88e4748e2..0ac81bf96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ -# **Upcoming release** - +# __Upcoming release__ + +# Release 1.15.2+dc.8 + +- Branch-sensitive assignment inference takes the body of + `if typing.TYPE_CHECKING:` (PEP 484). Names assigned only there, such as + class-body `TypeAlias` re-exports, resolve, and class bases read through + them enter the MRO. The test is followed through imports, attributes of + imported modules and a name's single plain assignment (typing_extensions' + `TYPE_CHECKING = typing.TYPE_CHECKING`), without inferring any object; other + tests, including `not TYPE_CHECKING`, stay undecided, and so does a name + written more than once (rebound, augmented, deleted, or assigned under + `global`/`nonlocal`). The constant is recognized as the `TYPE_CHECKING` + attribute of the module named `typing`, whatever its value, so a project + `typing` module binding `TYPE_CHECKING = False` still reads as true. The + decision is kept in the module's concluded data. + +# Release 1.15.2+dc.7 + +- Integrate the latest upstream master while retaining upstream 1.15.2 as the + published base. Preserve typed project APIs, UTF-8 function signatures, star + imports during inlining, and native try-statement regions. +- Bind a PEP 613 TypeAlias assignment to its value. +- Reject class method transfers when the destination exposes a mutable + builtins namespace during class construction. +- Resolve references in string-sequence declarations before later rebindings + or extensions of the referenced name. +- Moving a global rewrites a consumer's relative import of the moved name + (`from .origin import Name`, `from . import origin`) like the absolute form; + the stale relative import no longer survives the move. Imports through a + package initializer and aliased from-imports follow the move as well. +- `Project(..., config_files=False)` takes preferences only from the supplied + `prefs` over the defaults, without reading `pyproject.toml`, the ropefolder + `config.py` or the global configuration. +- `SourceBindingIndex.assignment_string_sequence` reads the binding Python + keeps: the last direct assignment of the name, extended by every later + `name += [...]` of literal strings. A bare annotation never rebinds it, and a + non-literal extension yields `()`. It previously returned the first + assignment, which diverged from the module's effective `__all__`. - #859 Fix `find_definition`/Rename not resolving `async def` methods called via `self` (@gadievron) - #895 patchedast Starred and keyword now consumes their syntactically expected and ** (@lieryan) - #896 patchedast cleanup and refactoring (@lieryan) @@ -7,16 +44,269 @@ - #922 Classify autoimport project sources by path containment (@marlon-costa-dc) - #926, #929 Raise RefactoringError instead of IndexError/ModuleSyntaxError when extracting an empty or whitespace-only region (@aliassar) - #933 Add type annotations to project.py (@lieryan) +- #920 Resolve inherited class attributes along the C3 method resolution order (@marlon-costa-dc) - #119, #928 Fix IndexError in rename and find occurrences when a `from` import is the last thing in a module without a trailing newline (@mcepl) - #921 Patch the else suite of try statements that also have a finally (@marlon-costa-dc) - #937 Preserve non-ASCII argument expressions and default values when changing function signatures (@Jinyong-Li) - #826, #940 Keep star imports in other modules when inlining a function or method (@chinghar) - #948, #949 Don't treat string prefixes such as the `f` in `f"..."` as occurrences when renaming a variable named `f`, `b`, `u`, `rf`, etc. (@aliassar) +# Release 1.15.2+dc.6 + +- A compound class base (`s[T]`, `pkg.Base`) resolves its root name as it is + bound at the class statement. A later facade alias (`s = Derived`) no longer + makes the class its own base, which recursed until `RecursionError`. + +# Release 1.15.2+dc.5 + +- Module-identity planning skips the write scan of function and class scopes + that declare no `global` names, whose writes cannot reach module identity: + 26% less time in `externaldata._plan_context` on large modules. + +# Release 1.15.2+dc.4 + +- `AutoImport.generate_cache` and `generate_modules_cache` start their worker + processes with `forkserver` (or `spawn` where it is unavailable) instead of + the platform default. Forking a process that runs other threads can + deadlock the child, and Python 3.12+ warns on every such fork. + +# Release 1.15.2+dc.3 + +- `MoveMethod.get_class_changes` proves that a class transfer keeps program + behaviour. Defaults, and annotations still evaluated eagerly, must be + literals. The destination's metaclass must be `type`. Its import-time code + must not rebind names through `globals()`, `vars()`, `locals()`, `exec`, + `eval` or the builtins module. A reference that runs before the destination + class exists is refused. +- A same-module transfer to an enclosing class now keeps module-level + bindings it reads, when the binding's single declaration is unchanged and + the method does not declare the name global or nonlocal. + +# Release 1.15.2+dc.2 + +- `SourceBindingIndex.context_for_node` spans only nodes of its own module's + AST. A node from another AST gets no context without reading this source; + before, its columns could fall inside a UTF-8 code point and raise + `UnicodeDecodeError` (non-ASCII modules, through `evaluate.eval_node`). + +# Release 1.15.2+dc.1 + +- Move the fork base to upstream release 1.15.2 (integrated by merge commit), + which carries the 1.15.1 DOA channel credential hardening in `runmod.py`. + The local suffix restarts at 1 for the new upstream base. +- Add `MoveMethod.get_instance_class_changes()` to transfer an + identity-decorated instance handler into a source class, rewriting + `self.member` accesses through explicit `receiver_rewrites`. +- Materialize function bodies of definition bindings on native query demand. +- Include the `annotations` type-hinting provider in the package. + +# Release 1.15.0+dc.12 + +- Never offer the literal pieces or format specs of an f-string as similar + finder matches: patchedast gives them no region, so a literal pattern equal + to one of them raised `AttributeError`. Expressions inside the replacement + fields still match. + +# Release 1.15.0+dc.11 + +- Memoize `Scope.get_region()` like `get_end()` and `get_logical_end()`, so + offset lookups stop re-scanning decorator tokens for every subscope. Scopes + are rebuilt with their module, so a changed source never sees a stale region. + +# Release 1.15.0+dc.10 + +- Tokenize a module once however many names are searched in it, caching the + name index by source text. + +# Release 1.15.0+dc.9 + +- Memoize offset resolution on each module until the project data changes, + so repeated occurrence searches no longer re-resolve the same offset. Any + change to a loaded resource retires every memo. + +# Release 1.15.0+dc.8 + +- Type `rope.base.utils.saveit` so decorated methods and properties, such + as `Project.pycore`, keep their return type for type checkers. +- Keep analysing classes whose hierarchy Python refuses to create, as upstream + does: their attributes, `super()`, `type()` and type hints follow the + declared bases. Only `AbstractClass.get_mro()` raises + `InconsistentHierarchyError`; the new `get_consistent_mro()` returns `None` + for such classes instead. +- Restore `rope.base.oi.type_hinting.utils.get_mro()`. +- Infer `super()` along the method resolution order and bind the real + receiver of calls made through it. +- Restore the upstream source form of lines that a linter had rewritten + without changing behavior, and keep comments in English. +- Infer assignments without source-index provenance, such as those of an + independently parsed expression, as unconditional assignments again. +- Store literal values in the object database the way their builtin base + class is stored, by resolving textual transformations along the MRO. +- Keep static object information for names read through a branch context that + does not select any of their assignments, and for value overlays of a + source name. +- Classify evaluated bases by the native `AbstractClass` contract rather than + equality with metatype markers. An unresolved base must retain its source + witness without being treated as a class supporting method resolution order. + +# Release 1.15.0+dc.7 + +- Resolve class attributes in C3 method resolution order instead of a + depth-first walk of the bases, so diamond hierarchies inherit the member + Python's `__mro__` selects. `AbstractClass.get_mro()` exposes that order. +- Model builtin classes with their runtime bases and `__mro__`, so `object` + sorts last and a builtin ancestor contributes only the attributes it + defines itself. +- Select inherited docstring and assignment type hints along the same + method resolution order instead of a breadth-first walk. +- Keep bases rope cannot evaluate to a class as `UnresolvedBase` markers in + `AbstractClass.get_bases()` and in the method resolution order instead of + dropping them; `get_superclasses()` still returns only the classes. +- Raise `InconsistentHierarchyError` for cyclic, repeated, and unmergeable + bases, which Python refuses to create. + +# Release 1.15.0+dc.6 + +- Classify evaluated bases by the native `AbstractClass` contract rather than + equality with metatype markers. An unresolved base must retain its source + witness without being treated as a class supporting method resolution order. + +- Infer `super(cls, obj)` along the method resolution order of `cls` past + `cls` itself instead of as an instance of its first base. Methods and + classmethods called through it bind the real receiver. + +# Release 1.15.0+dc.7 + +- Resolve class attributes in C3 method resolution order instead of a + depth-first walk of the bases, so diamond hierarchies inherit the member + Python's `__mro__` selects. `AbstractClass.get_mro()` exposes that order. +- Model builtin classes with their runtime bases and `__mro__`, so `object` + sorts last and a builtin ancestor contributes only the attributes it + defines itself. +- Select inherited docstring and assignment type hints along the same + method resolution order instead of a breadth-first walk. +- Keep bases rope cannot evaluate to a class as `UnresolvedBase` markers in + `AbstractClass.get_bases()` and in the method resolution order instead of + dropping them; `get_superclasses()` still returns only the classes. +- Raise `InconsistentHierarchyError` for cyclic, repeated, and unmergeable + bases, which Python refuses to create. + +# Release 1.15.0+dc.6 + +- Resolve class-owned PEP 695 aliases from other lazy aliases in the same class, + including forward declarations and names shadowing module aliases. Conditional + declarations retain their ambiguous source identity. +- Resolve opaque annotation references through their captured source contexts + when refactoring checks request them, avoiding duplicate binding resolution + while constructing the source index. +- Add `match_filter` and `import_filter` to `Restructure`: callables consulted + for every match found in a module and for the imports added to a changed + module, so clients select both without overriding private methods. + +# Release 1.15.0+dc.5 + +- Record PEP 695 type-alias declaration locations in module, function, and class + scopes without evaluating or substituting the lazy alias value. +- Preserve first-binding locations across ordinary assignments, repeated aliases, + imports, and native renames; verify these contracts against the installed wheel. +- Keep constructor and method aliases local to their declaration scope, without + polluting instance attributes collected by the target-only visitor. +- Select native setup interpreters independently of ancestor project constraints. + +# Release 1.15.0+dc.4 + +- Patch expressions in the `else` suite before `finally` for both `try` and + `try`/`except*`, preserving source regions and semantic matching. +- Verify source round-tripping and parenthesized attribute matching against the + installed wheel as well as the source checkout. +- Initialize the documented source-folder configuration without invoking a + deprecated private helper, preserving explicit folder selection without warnings. +- Preserve f-string references across supported Python tokenizers and expression + coordinates for escaped braces and Unicode, adopting the reviewed native fixes. +- Keep autoimport project-relative paths and nested installation roots separate on + Windows, and scope the metadata task check to its required checks permission. +- Validate published downstream GitHub Release assets through the native installed + wheel contract, without sending local-version builds to the upstream PyPI publisher. +- Isolate installed-wheel pytest configuration from ancestor projects and inherited + pytest options while retaining the native test configuration and contracts. + +# Release 1.15.0+dc.3 + +Flext-ecosystem compatibility and robustness over 1.15.0+dc.2: + +- Expose a module scope over builtin modules (`BuiltinModule.get_scope` + `BuiltinModuleScope`), so semantic-module consumers recognize C/builtin modules +- Do not require callers to preseed `ignored_resources` when a python path folder inside the root contains site-packages (#756 follow-up); both behaviors carry regression tests + +# Release 1.15.0+dc.2 + +Performance and inference cycle on the 1.15.0+dc.1 line: + +- Fix property inference regression from the parameter-bindings adoption: record function calls without resolving the function from its own text, restoring typed attributes on properties (rename no longer misses subclass call sites) +- Resolve external class-data plans once per module entry: single shared plan store, per-owner lazy resolution, attribute readers grouped in one pass, class-binding scan memoized +- Cut the source-binding index build cost (typed node dispatch, slotted contexts, ASCII fast paths in ASTLinesAdapter) +- Skip the opaque-reference scan for modules that never spell the renamed name and modules without class-attribute writes + +# Release 1.15.0+dc.1 + +Fork of upstream rope 1.15.0: adopts the upstream open pull requests +below and adds the datacosmos feature branches. + +Upstream pull requests adopted: + +- #756 Fix import paths with `site-packages` as part of package name (@last-partizan) +- #790 Fix multiline empty tuple with comment (@apmorton) +- #808 Handle the edge case where an f-string looks like `f"#{some_var}"` (@rlbr) +- #814 Treat packages in `ignored_resources` as non-project files for autoimport (@smheidrich) +- #836, #837 Correct `_const_node_type_names` fallback for Python 3.14+ (@EhteshamSid) +- #848 Fix autoimport.sqlite deriving module names from non-package folders (@deepakganesh78) +- #858 Don't fold source after an f-string with an unmatched literal bracket, Python 3.12+ (@gadievron) +- #859 Recognise `async def` method headers in worder (@gadievron) +- #861, #865 Recognize match pattern captures as assignments (@yangfan-yf-yf) +- #862, #874 Treat type alias statements as name assignments and refuse inlining them (@TanbirRamim) +- #867 patchedast: raise MismatchedTokenError instead of AttributeError (@quazardous) +- #868 patchedast: don't match tokens inside string literals (@quazardous) +- #864, #866 Preserve compound assignments when inlining (@onuracar-dev) +- #876 Fix extraction of nested named expressions (@yangfan-yf-yf) +- #877 Preserve used imports during filtered import cleanup (@yangfan-yf-yf) +- #878 Define functions extracted from a class body before the class (@ethanstoner) +- #881 Preserve original names in external from-imports (@ryanchou1994) +- #879, #880 Fix patchedast region for implicitly-concatenated f-strings (@stevemadere) +- #802, #887 Fix source regions for regular argument annotations (@Strohutt) +- #886, #889 autoimport: track and close SQLite connections across worker threads (@mcepl) +- #827, #892 Preserve subscript index reads during method inlining (@SAY-5) +- #893 Resolve function-local constructor names through their scope (@SAY-5) +- #894 Refuse unsafe inlining into deferred annotations and type parameters (@yangfan-yf-yf) +- #897 Refuse unsafe inlining of lambda-local bindings (@yangfan-yf-yf) +- #898 Preserve positional-only and keyword-only parameter bindings (@yangfan-yf-yf) +- #899 Keep unresolved call keyword labels separate from variables (@yangfan-yf-yf) +- #900 Preserve annotation, lambda and comprehension source bindings (@yangfan-yf-yf) +- #901 Fix field renames after external class data writes (@yangfan-yf-yf) +- #902 Fix namespace package directory renames (@yangfan-yf-yf) +- #903 Preserve initializer behavior when removing inlined variables (@yangfan-yf-yf) +- #904 Bind receivers for ordinary functions stored on classes (@yangfan-yf-yf) +- #905 Ignore `node_modules` directories by default (@marcoeangeli) + +Fork contributions (also proposed upstream): + +- #844 Detect the JoinedStr end quote from the terminating delimiter (@marlon-costa-dc) +- #882 Match MatchSequence delimiters by tokenizing the node segment (@marlon-costa-dc) +- Find textual occurrence offsets through the tokenizer so rename locates names inside PEP 701 f-strings +- Walk positional-only, keyword-only and annotated parameters in patchedast signatures +- Guard type-hint providers against non-AssignedName pynames and infer `X | Y` union annotations +- Compute scope regions from native AST positions, preserving Unicode identifiers, decorators, and LF, CRLF, and CR newlines +- Honor namespace-package precedence when resolving imports across source folders +- Route project, resource, and observer filesystem queries through the fscommands owner (exists/isfile/isdir/listdir/stat/realpath) +- Model namespace packages as empty modules so scope machinery can walk them +- Discover the compiled extension module used by autoimport tests from the import system + +Datacosmos internal: + +- Project(external_project=...) and NoProject(fscommands=...) share one captured filesystem so native module resolution survives source-tree changes + # Release 1.15.1 -- #xxx Change runmod.py to pass DOA channel credential in a more secure manner, - see GHSA-64m2-4gp9-7cqx/CVE-XXX for details (@AliAAlMousa, @lieryan) +- Change runmod.py to pass DOA channel credential in a more secure manner, + see GHSA-64m2-4gp9-7cqx (@AliAAlMousa, @lieryan) # Release 1.15.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 3d982a324..36f040a5d 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -116,7 +116,7 @@ the community. This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). diff --git a/FORK-TRACKING.md b/FORK-TRACKING.md new file mode 100644 index 000000000..b73aedac8 --- /dev/null +++ b/FORK-TRACKING.md @@ -0,0 +1,51 @@ +# Fork tracking + +Open follow-ups for the `marlon-costa-dc/rope` fork. Update this file whenever an item changes. + +## Working rule + +- External PRs are followed until their cycle ends (merged or closed). +- Only remote branches are kept. A local worktree is removed as soon as its work is pushed, and recreated when the branch needs work again. +- Unique local work is first pushed to a `preserve/*` ref, judged, then adopted or retired. + +## Fork state + +- `main` mirrors `python-rope/rope:master`; the scheduled workflow lives on the + fork's default branch (`master`). +- `dc-use` is the downstream integration branch. Use the full producer commit + or the artifact SHA-256 when comparing installations. +- The published [1.15.2+dc.6](https://github.com/marlon-costa-dc/rope/releases/tag/1.15.2%2Bdc.6) + identifies `41b96032`; the later consumer Git pin `4cb40a6f` also declares + that version. The next downstream package uses a new local version. + +## Upstream contributions + +Merged: [#845](https://github.com/python-rope/rope/pull/845), +[#921](https://github.com/python-rope/rope/pull/921), and +[#922](https://github.com/python-rope/rope/pull/922). +[#938](https://github.com/python-rope/rope/pull/938) was closed; any upstream +follow-up should be split by behavior for review. + +Open with the maintainers: + +- [ ] python-rope/rope#920 — Resolve inherited class attributes along the C3 method resolution order (`pr/c3-mro`) +- [ ] python-rope/rope#919 — Type saveit so decorated members keep their return type (`pr/typed-saveit`) +- [ ] python-rope/rope#918 — Add match_filter and import_filter hooks to Restructure (`pr/restructure-hooks`) +- [ ] python-rope/rope#911 — Route filesystem queries through the fscommands owner (`pr/fscommands-routing`) +- [ ] python-rope/rope#910 — Honor namespace-package precedence when resolving imports (`pr/namespace-precedence`) +- [ ] python-rope/rope#909 — Compute scope regions from native AST positions (`pr/scope-regions`) +- [ ] python-rope/rope#908 — Guard type-hint providers and infer X | Y union annotations (`pr/type-hinting-guards`) +- [ ] python-rope/rope#907 — Walk positional-only, keyword-only and annotated parameters in patchedast (`pr/signature-tokens`) +- [ ] python-rope/rope#906 — Find textual occurrence offsets through the tokenizer (`pr/occurrences-tokenizer`) +- [ ] python-rope/rope#882 — patchedast: locate multi-line f-strings, replacement fields and match-sequence delimiters by parser coordinates (`pr/parser-coordinates`) +- [ ] python-rope/rope#844 — Fix patchedast aborting on ''' inside a double-quoted f-string (`fix/joinedstr-triple-quote-in-body`) + +## Consumer propagation + +Update the Rope pin in FLEXT Infra's codegen configuration and propagate it +through the native generation and upgrade flow. Verify the installed producer +commit and run the consumer refactorings from the new environment. A lockfile +alone does not prove what a consumer imports. + +Keep historical `datacosmos` and `Sync` refs until dependent locks and residual +contracts have been reconciled. The MCB lock still refers to `datacosmos`. diff --git a/MANIFEST.in b/MANIFEST.in index c6e77775f..8309d3c18 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ include README.rst COPYING setup.py MANIFEST.in CHANGELOG.md ropetest-package-fixtures/external_fixturepkg/dist/external_fixturepkg-1.0.0-py3-none-any.whl ropetest-package-fixtures/external_fixturepkg/dist/external_fixturepkg-1.0.0.tar.gz recursive-include rope *.py +include rope/py.typed +include Makefile recursive-include docs *.rst recursive-include ropetest *.py diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..2d8eb8cf1 --- /dev/null +++ b/Makefile @@ -0,0 +1,84 @@ +VENV := $(shell dirname "$(CURDIR)")/.rope-venvs/$(shell basename "$(CURDIR)") +PYTHON := $(VENV)/bin/python +UV ?= uv +BOOTSTRAP_PYTHON ?= 3.13 +TEST_PROFILE ?= +TEST_WORKERS ?= 2 +EDITABLE_MODE ?= +export PYTHONDONTWRITEBYTECODE := +export PYTHONPYCACHEPREFIX := $(VENV)/pycache +export COVERAGE_FILE := $(VENV)/.coverage + +.PHONY: setup test test-file check check-codecov fmt build artifact-check source-ancestry source-fixture + +source-fixture: + "$(PYTHON)" -m ropetest.source_ancestry_fixtures + +source-ancestry: + test -n "$(REQUEST)" + "$(PYTHON)" -m ropetest.source_ancestry_probe "$(REQUEST)" + +setup: + @set -eu; \ + expected_python="$$( $(UV) python find --no-project --system "$(BOOTSTRAP_PYTHON)" )"; \ + if [ -e "$(VENV)" ] || [ -L "$(VENV)" ]; then \ + if [ -L "$(VENV)" ] || [ ! -x "$(PYTHON)" ] || [ ! -f "$(VENV)/pyvenv.cfg" ]; then \ + printf 'ERROR: existing environment is not a physical venv: %s\n' "$(VENV)" >&2; \ + exit 2; \ + fi; \ + if ! "$(PYTHON)" -c 'from pathlib import Path; import sys; sys.exit(0 if Path(sys.executable).resolve() == Path(sys.argv[1]).resolve() else 2)' "$$expected_python"; then \ + printf 'ERROR: existing environment uses another Python interpreter: %s\n' "$(VENV)" >&2; \ + exit 2; \ + fi; \ + else \ + $(UV) venv --python "$$expected_python" "$(VENV)"; \ + fi + $(UV) pip install --python "$(PYTHON)" $(if $(EDITABLE_MODE),--config-setting 'editable_mode=$(EDITABLE_MODE)') -e '$(CURDIR)[doc,dev]' + +test: + @set -- --cov -v -ra -o cache_dir="$(VENV)/pytest-cache" \ + --numprocesses="$(TEST_WORKERS)" --dist=loadscope --max-worker-restart=0; \ + if [ -n "$(TEST_PROFILE)" ]; then \ + "$(PYTHON)" -c 'import cProfile, pstats, runpy, sys, traceback; exec("\n\ +profiler = cProfile.Profile()\n\ +profile_path = sys.argv.pop(1)\n\ +original = None\n\ +try:\n\ + profiler.runcall(runpy.run_module, \"pytest\", run_name=\"__main__\", alter_sys=True)\n\ +except BaseException as error:\n\ + original = error\n\ + raise\n\ +finally:\n\ + try:\n\ + profiler.dump_stats(profile_path)\n\ + with open(profile_path + \".txt\", \"w\", encoding=\"utf-8\") as report:\n\ + pstats.Stats(profile_path, stream=report).sort_stats(\"cumulative\").print_stats()\n\ + except BaseException as publication:\n\ + if original is None or (isinstance(original, SystemExit) and original.code in (None, 0)):\n\ + raise\n\ + traceback.print_exception(type(publication), publication, publication.__traceback__)\n\ +")' "$(TEST_PROFILE)" "$$@"; \ + else \ + "$(PYTHON)" -m pytest "$$@"; \ + fi + +test-file: + test -n "$(FILE)" + "$(PYTHON)" -m pytest -v -ra -o cache_dir="$(VENV)/pytest-cache" "$(FILE)" + +check: + "$(PYTHON)" -m pre_commit run --all-files + +check-codecov: + curl --fail-with-body --data-binary @.github/codecov.yml https://api.codecov.io/validate + +fmt: + "$(PYTHON)" -m pre_commit run black --all-files + "$(PYTHON)" -m pre_commit run isort --all-files + +build: + "$(PYTHON)" -m build "$(CURDIR)" + $(MAKE) artifact-check + +artifact-check: + "$(PYTHON)" "$(CURDIR)/ropetest/package_artifact_contract.py" diff --git a/bin/tag-release.py b/bin/tag-release.py index f0b6340f3..4c8285d84 100644 --- a/bin/tag-release.py +++ b/bin/tag-release.py @@ -15,7 +15,7 @@ def get_changelog(version): search_heading = f"# Release {version}" curdate = datetime.today().date().isoformat() - headings = re.findall(r"# Release \d+\.\d+\.\d+", s) + headings = re.findall(r"# Release \d+\.\d+\.\d+(?:\+[\w.]+)?", s) version_heading_index = headings.index(search_heading) version_changelog = s[ s.index(headings[version_heading_index]) : s.index( diff --git a/docs/README.md b/docs/README.md index 89c42db65..b5474116c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -### Notice +# Notice If you got to this page on Github accidentally, this is the source code for rope's documentation. You probably want to use the pre-rendered version of diff --git a/docs/contributing.rst b/docs/contributing.rst index 796699986..54095ccd6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -92,16 +92,36 @@ more specific to Traad. Source Repository ================= -Rope uses GitHub_. The repository exists at -`python-rope/rope`_. +Rope uses GitHub_. The upstream repository is `python-rope/rope`_. +This guide's fork automation is maintained in `marlon-costa-dc/rope`_. + +.. _`marlon-costa-dc/rope`: https://github.com/marlon-costa-dc/rope Setting up for local development ================================ -#. Clone repository: ``git clone https://github.com/python-rope/rope.git`` -#. Create a virtualenv: ``python -m venv rope-venv`` -#. Activate the virtualenv -#. Install the project into the venv: ``pip install -e '.[doc,dev]'`` +Clone the fork's ``dc-use`` integration branch, which contains this Makefile:: + + git clone --branch dc-use https://github.com/marlon-costa-dc/rope.git + cd rope + +On a POSIX system: + +#. Install Make and ``uv``. +#. Run ``make setup`` from the checkout root. + +The fork's Makefile owns the physical development environment outside the +checkout. ``BOOTSTRAP_PYTHON`` selects its interpreter; its default is declared +in that Makefile. Use the same root Makefile for the checks described below. +Setup installs the project's ``doc`` and ``dev`` extras. An existing environment +with another interpreter is rejected before installation. + +On native Windows, use Python's ``venv`` module to create a separate environment +outside the checkout, select that environment's interpreter, then run +``python -m pip install -e ".[doc,dev]"`` from the checkout root. The native +``python -m pytest`` workflow in ``.github/workflows/main.yml`` runs on Windows; +the POSIX Makefile is not its launcher. Both platforms consume the same project +extras and test suite. Submitting pull requests ======================== @@ -128,9 +148,47 @@ Programming Style Testing ------- -Rope uses `pytest`_. To run the test:: - - pytest -v +Rope uses `pytest`_. On POSIX, the fork's root Makefile prepares its own physical +development environment and runs the complete suite with coverage:: + + make setup + make test + +The test recipe distributes the complete inventory across the bounded worker +count declared by ``TEST_WORKERS`` in the Makefile. ``loadscope`` keeps each +class or module's tests together. Workers own their temporary projects and +fixture environments; the controller publishes the combined coverage and +persistent test cache. Automatic worker restarts are disabled, so a crashed +worker fails the invocation without rerunning its tests. + +Before collection, the pytest session selects the public multiprocessing +``spawn`` context. This keeps process-based tests safe inside multithreaded +xdist workers on the supported Python versions. It does not change the SDK's +process policy or claim coverage of production's fork context. The native +recipe starts a fresh interpreter; a process that fixed its context earlier +is rejected before collection rather than having that context forcibly replaced. +Timeout limits +use the installed pytest-timeout marker directly, so worker startup cannot +replace a test body with a compatibility decorator. + +The environment and test cache live outside the checkout. For a diagnostic +profile of that same invocation, pass an explicit output path whose parent +directory already exists:: + + make test TEST_PROFILE=/absolute/path/to/existing-directory/test.pstats + +The invocation writes the raw profile and a complete cumulative-time report at +the same path with ``.txt`` appended. The text report reads the saved raw profile. + +Profiling is disabled by default and does not change test selection, coverage, +or cache configuration. The profile covers the controller's pytest module, +including worker coordination and pytest teardown; it does not cover worker +execution, other subprocess internals, or interpreter shutdown after the module +exits. Original exceptions and nonzero exits propagate unchanged. A profile +publication failure fails an otherwise successful invocation. When execution +already failed, its secondary publication traceback is printed without replacing +the original failure or changing its exception chain. An interrupted or incomplete run +remains a failed diagnostic, even when a profile was saved. Many of rope's tests are still written using ``unittest.TestCase`` style, but running the test suite using @@ -140,10 +198,6 @@ Make sure to have complete test suite passing and add new tests for the changes you are providing with each new submission. -All required packages for development could be installed with:: - - pip install -e ".[dev]" - .. _GitHub: http://github.com/ .. _`python-rope/rope`: https://github.com/python-rope/rope .. _`pull request`: https://help.github.com/articles/using-pull-requests diff --git a/docs/library.rst b/docs/library.rst index 6a62c3e00..b460823db 100644 --- a/docs/library.rst +++ b/docs/library.rst @@ -14,6 +14,325 @@ Quick Start This section will help you get started as soon as possible. +Formatted string expressions infer the native ``str`` instance type without +evaluating their formatted values. Their nested expressions remain visible to +the existing reference, call and reflection analysis. + +Semantic match capture regions +------------------------------ + +Matches returned by ``rope.refactor.similarfinder.SimilarFinder.get_matches`` +provide ``get_capture_region(name)``. It returns the captured variable's +half-open character range in the original source, for both expression and +statement patterns. An unknown capture raises ``ValueError``. Clients can +therefore use source regions without depending on Rope's syntax nodes. + +The module's existing ``get_source_binding_index().absolute_import_roots()`` +returns a frozen set of absolute top-level import names from its indexed +declarations, including guarded and nested imports. Relative imports are +excluded. This query neither imports the declared packages nor resolves their +classes, and it preserves declarations whose local bindings are overwritten. +Occurrence import roles use the same native import-declaration spans. A +``yield from`` before a later import must not classify an intervening function +declaration as imported and silently omit its signature change. + +``assignment_string_sequence(name, scope=None, *, class_metadata=False)`` reads the last direct +assignment and subsequent literal ``+=`` extensions in the indexed module or +an explicitly supplied class scope. It +returns a tuple for a literal string list, tuple or set, and follows module +name references and single-name ``tuple``/``list``/``set``/``frozenset`` wrappers +at the position of the referring assignment. +Missing declarations return ``None``; present empty or unsupported values return +an empty tuple. Cyclic references raise ``ValueError``. It uses the module's +existing parse without evaluating authored code or resolving class ancestry. +A supplied scope must belong to that same indexed module; another module's +scope raises ``ValueError`` instead of mixing its declarations with local names. +With ``class_metadata=True``, an absent module declaration may be read from a +single directly declared class. Module declarations still take precedence, +including empty or unsupported values. This mode does not resolve class scopes +or superclasses and cannot be combined with an explicit scope. + +Class headers use the binding that exists before the new class replaces its +name. In particular, importing ``Foo`` and then declaring ``class Foo(Foo)`` +inherits the imported class; it does not declare a self-inheritance cycle. +For a generic class, a same-named native type parameter takes precedence over +that earlier binding, as it does for the other names in the generic header. +This source-binding requirement does not waive rejection of an inconsistent +hierarchy or introduce a second method-resolution-order implementation. + +``get_source_binding_index().class_layout(scope)`` exposes the source layout of +a class in that same module. Its immutable result contains the declaration, +header and body regions, the offset immediately after the declaration colon, +the name region and base-list insertion offset, optional +parenthesized argument-content region, direct keyword/base/decorator regions, +original base bindings, named type-parameter regions, +and direct annotated +bindings. Each annotated binding retains its complete statement region, the +native binding of its annotation qualifier and whether a value is present. +Qualifier resolution uses the annotation's original source context, including +aliases and generic header bindings. An unresolved qualifier remains ``None``; +it is not replaced by a guessed name. Regions are half-open character offsets +into the indexed module, not lines or a copy of its syntax tree. A scope from +another module, or a non-class scope, is rejected. Consumers can use these facts +and the existing ``context_at``/native class queries without inspecting syntax +nodes, reconstructing a header or opening a second project. +The argument region distinguishes ``class A:`` from ``class A():`` and preserves +keyword-only headers such as ``class A(metaclass=M):``. Header tokens are read by +Rope's existing lexical reader over the bounded header, so Unicode identifier +spelling, comments, generic bounds and explicit line continuations keep their +original character offsets. A leading decorated member belongs to the body, +including its decorator lines. + +``assignment_layouts(scope=None, *, name=None)`` returns immutable metadata for +direct assignments and type-alias declarations in the supplied scope (the module +by default). Each record carries its native statement kind, complete statement +region, named target regions, optional value/annotation regions and the native +value binding resolved in its original source context. ``root_binding`` is the +binding of a bare-name root below attribute access, or ``None`` for other +expressions. ``string_sequence`` is the exact tuple of a direct literal string +list, tuple or set, or ``None`` for another value. ``type_parameters`` retains +named parameter regions of generic type aliases. Value inference is explicit: +``resolve_values=True`` resolves ``value_binding`` and ``root_binding``; the +default returns layout only. A name selection is applied before this inference, +so a query for one binding does not infer unrelated declarations. Multiple writes remain separate ordered records; +clients retain responsibility for admitting ambiguous or augmented writes. +Like ``class_layout``, this query uses only the existing indexed source and +rejects a scope from another module. + +The existing ``SourceContext.access`` query distinguishes expression reads, +writes and deletions (``"read"``, ``"write"`` and ``"delete"``), returning +``None`` for a context that is not a name/attribute/subscript access. Clients +can combine this with the original context environment and semantic match +regions without interpreting a syntax node. + +``decorator_bindings(scope)`` returns the ordered native callee bindings of a +source declaration's decorators. A called decorator resolves its callable in +the original header environment rather than evaluating its factory result. +Declarations without decorators return an empty tuple. Unknown callees retain +``None`` and native resolution errors propagate; no decorator is imported or +executed by this query. + +``string_mapping_arguments(callee, position)`` locates a named call's positional +mapping argument in the existing index. + +``import_layouts()`` exposes imports from that same index, including guarded and +local imports. Each immutable record retains the statement region, optional +from-provider region, module name, relative level and original imported names, +aliases and their source regions. It never hoists imports or resolves unrelated +providers. Native lexical offsets preserve the original importing boundary. +Alias-separator regions identify the existing commas between indexed aliases; +consumers can retain intervening comments when removing selected names. + +The string-mapping query follows a single native assignment +for a referenced lexical name and retains native bindings of the selected callee +and single-argument wrappers. Each result carries the mapping region and literal string keys with +string-or-string-tuple values and the original value-token regions. Unsupported +or ambiguous shapes have ``entries=None``; clients decide whether that is an +expected refusal. The query neither evaluates a wrapper nor assigns it mapping +semantics. It opens no project, parses no new source and keeps unrelated strings +out of the selected mapping. + +Subscription analysis must distinguish instance ``__getitem__`` from class +``__class_getitem__`` and retain the actual subscription argument. A metaclass's +``__getitem__`` has precedence over the class hook. An unresolved subscription +remains unresolved; substituting its origin class without proving the hook's +result is not an ancestry contract. Native support for these cases is being +qualified against source-derived regressions before consumers rely on it. + +The standard-library ``Generic`` and ``Protocol`` alias model applies to builtin +and source implementations across supported Python versions. Source declarations retain +their native class identity: admission authenticates the top-level declaration +against the runtime library's name and physical module resource. A project-local +``typing.py`` or a same-named class does not select this model. The shared alias +factory retains every argument and recognizes only the type-variable capabilities +available in that interpreter; unavailable APIs do not become substitutes or skips. +When that authenticated owner is ``Generic`` or ``Protocol``, invalid parameters +raise the original ``TypeError`` before an alias is constructed. Concrete aliases +such as ``Mapping[str, str]`` have a different owner and do not justify admitting +invalid ``Generic`` or ``Protocol`` arguments. This preserves the +`typing specification's parameter contract +`_; +an inference failure must be diagnosed at its actual source rather than bypassing +this validation. The rejecting exception retains native object and type identities +for the supplied arguments, including their declaring module resources when known. +Unknown objects are reported by their native kind without assuming a class name; +the diagnostic reads the values already supplied to the alias factory and does not +resolve another namespace. Diagnostic failures remain secondary notes on the same +``TypeError``. Restoring the validation alone does not qualify a consumer. +Source ``Generic`` bases use the same native C3 machinery and runtime-library base +identities as their builtin counterpart. This contract does not authorize importing +application targets or replacing an unresolved custom hook with its origin. + +Variadic generic parameters use the authenticated standard-library ``Unpack`` +operation and retain their inner ``TypeVarTuple`` by reference. A raw +``TypeVarTuple`` is not itself a valid ``Generic`` or ``Protocol`` parameter. +Interpreters lacking these APIs retain their real missing-import and unresolved +binding contracts; an alternate provider or a skipped positive case does not +qualify this capability. + +The one-argument builtin ``type(value)`` operation is distinct from Rope's +``PyObject.get_type()`` kind markers. Class operands select their actual native +metaclass; instance operands select their known native class. Unresolved operands +and metaclass witnesses remain unresolved. This does not change the existing +three-argument class-construction analysis or make a metaclass instance into a +source-defined class. + +Method-transfer declaration reads authenticate the native receiver getter and +the member's descriptor behavior. Plain scalar members are recognized by their +native builtin class identity; absent literal-value metadata is not a known +``None`` value and cannot admit an arbitrary descriptor. Inherited class members, +direct module declarations and their existing binding checks remain unchanged. +The instance-transfer ``resolve_binding`` dependency defaults to native +``evaluate.eval_node``. A caller may supply its existing source-view resolver; +the same dependency resolves original and proposed expressions, while native +getter, descriptor and binding-identity checks remain mandatory. + +Native class-construction analysis must retain the supplied class name, ordered +bases, explicit metaclass and known namespace bindings in the existing +``AbstractClass`` model. Empty bases select the native ``object`` class; +ancestry and inherited attributes use native C3. Literal values must retain +their payload through ordinary argument inference while preserving their +builtin type identity. An unresolved name, base or namespace must not become an +invented class. Construction uses source inference without importing the +application or executing a runtime class factory. This contract includes +``type.__new__``, three-argument ``type`` and source metaclass calls whose +constructor resolves to native ``type.__new__``; qualification against real +consumers is pending. + +Calling a class whose ``__new__`` is a source Python function with implicit or +explicit staticmethod semantics supplies the requested class exactly once through +the existing native argument binder. Simple names and qualified class calls use +the same path. Its inferred return is retained, including an unresolved return; +the source constructor does not manufacture an instance of its declaring proxy. +The native ``type.__new__`` path remains unchanged. This cut does not alter the +existing classmethod or lambda constructor paths and does not establish complete +constructor qualification. The original public consumer remains the runtime +acceptance boundary before SDK regression tests and integration. + +While an assignment's right-hand side is inferred, a recursive read of that same +``AssignedName`` observes only its preceding assignments on a compatible control +path. The existing source-binding index records nested ``if`` arms while visiting the +condition and both suites in source order. Literal conditions select their +reachable arm; opaque conditions retain both outcomes, including a path without +a binding. Only outcomes referring to the same object join as that object; +different or unresolved outcomes remain unknown. A join of distinct outcomes +remains unknown before annotation inference; an ordinary unresolved right-hand +side retains the existing annotation contract. Recursive prefix inference retains +an ambiguous witness when its active right-hand side or a preceding candidate +depends on loop, exception or pattern-selection control that has not been modeled. +Nonrecursive inference retains the existing treatment of those controls. +Ambiguity propagates through recursive reads of that same binding during the +current inference, so a surrounding assignment cannot hide it with an annotation. +Dependent names use the active expression's explicit ``SourceValueScope`` branch +context, rather than a cached result that joins the entire scope. Contextual +values retain a reference to the canonical source name and its definition. +Branch facts belong to that expression and structural owner; another function +call starts from its own native parameter epoch, without ambient transport. +Decorators, headers and defaults retain their creator's branch context; function +and lambda bodies begin a separate control context. Assignment metadata comes +from that same source-index pass, including header, walrus and comprehension +bindings. An assignment without source-index provenance, such as one in an +independently parsed expression, carries no branch facts and keeps Rope's +existing unconditional inference. A recorded loop, exception or +pattern-selection context likewise keeps its existing nonrecursive inference +semantics. This dependency-context +capability is under qualification and does not yet establish consumer acceptance. + +Reassigning a formal parameter retains its original ``ParameterName`` in function +scope, headers and keyword bindings. Source assignments are separate value facts +holding that parameter by reference; raw parameter values, passed-object queries +and definition locations keep their original contracts. ``SourceValueScope`` +looks up those facts at the exact source ``Name`` node and returns a +``SourceValueName`` whose source identity is the original parameter. A read sees +only writes whose real right-hand side ends before the read starts; its own +right-hand side and future writes are excluded. An annotation without a value is +not a parameter write. Header and walrus writes join the same ordered facts +without replacing the formal binder. Only a proven path with no preceding write reads the +current argument value; missing provenance, unresolved writes and ambiguous +paths stay unknown. These contextual values are inferred without populating the +full-binding cache, and another native call supplies its own argument epoch. +For a generic function or a method of a generic class, the source context's +structural scope authenticates the owning function. Its lexical environment is +retained separately so type parameters remain visible during contextual reads. +Occurrence discovery unwraps only native ``SourceValueName`` instance views, +including nested views, before querying the original parameter's passed objects. +The original parameter identity and ordinary instance inputs remain unchanged. +The inference boundary, arm context and ambiguity state are restored on exit, +including failures; a prefix value never replaces +the full binding's cached object. This preserves native name and definition +identity, the existing treatment of unknown values and cycles, and annotation +inference. It does not select a convenient known branch or invalidate other module +facts. + +Module-level assignments to source-function attributes belong to the existing +native module event pass. Their receiver and value must use the bindings in +effect at that event, retaining object identity through aliases, later shadowing +and reassignment. An attributed function remains a function: protocols such as +``__mro_entries__`` consume its assigned attribute rather than a substituted class. +During module construction, unresolved lexical lookups reach the native builtin +namespace without reopening the module's still-in-progress attribute inventory. +Conditional or ambiguous writes retain an explicit unresolved witness; a missing +attribute cannot be manufactured to hide an unproved write. This capability must +reuse the native module and binding lifecycle, without a second source scan, +attribute index or persistent cache. Qualification against real consumers is +pending; the declaration of this contract does not attest implementation. +The current event capture covers a single direct, unconditional assignment to an +attribute of a directly declared module function. Alias receivers, conditional +writes and chained targets still require the temporal binding capability and +separate qualification; this initial cut does not establish their acceptance. +If a known function's attribute is overwritten by an unsupported conditional or +chained event, that event replaces the previous value with an existing native +``EvaluatedName`` carrying its module and line. Consuming it raises ``RopeError`` +with that event's origin instead of publishing the old value as definitive. +A later supported unconditional write can replace this unresolved event. + +When evaluating a declared base raises, Rope re-raises that same exception +with its original traceback. Its ``__notes__`` metadata identifies the source +module, declaring class, base line and base expression; nested failures retain +each declaring context. The metadata is stored on Python 3.10 as well, without +requiring ``BaseException.add_note``. Python 3.11 and later render these notes +with the traceback. This diagnostic neither substitutes a base nor establishes +that the ancestry or a consuming refactoring succeeded. If formatting the source +context itself fails, that secondary failure is recorded as a note without +replacing the original exception. + +To diagnose a real source graph without importing its application, run +``make source-ancestry REQUEST=/absolute/request.json``. The request provides +``root``, ``python_path`` (a list of source search directories), and ``targets`` +(a list of objects with ``module`` and ``attributes``). The native project reports +its elected resources, bases and unresolved expression witnesses as JSON. For +unresolved bases it identifies the inferred source binding and whether that +object exposes the native ``__mro_entries__`` protocol. For source subscription +hooks it also reports native parameters, local assignments, +return expressions and directly assigned source-callable frames. These frames +describe inferred values; they do not execute the analyzed package. +Subscription-hook frames also compare recorded writes to formal parameters with +the effective native binder and the inferred right-hand-side value, using the +existing source index rather than another traversal. +They report individual source-position reads separately from raw formal values, +so retaining a parameter's identity cannot be mistaken for losing its writes. +An optional ``frames`` list, with the same module/attributes addressing, reports +selected source-function frames after ancestry inference in that same project, +including each local assignment expression's inferred value. +For a selected frame, ``preassignment_names`` optionally compares the original +binding, its individually invalidated inference, and the last assignment under +a native temporary scope containing only earlier assignments of that name. +This controlled diagnostic supplies no expected object and changes no source; +its result is separate from the unmodified ancestry witness. +Search directories are analysis inputs, not additions to the diagnostic interpreter's +``sys.path``. Native errors propagate instead of substituting another provider. +An optional ``bindings`` list addresses native symbols in the same way, without +requiring class ancestry. Assigned call expressions report their receiver, +callable, arguments and inferred result, allowing an unresolved factory boundary +to be inspected without swallowing the failing ancestry operation. + +``make source-fixture`` executes only the maintained diagnostic example +``ropetest.source_ancestry_fixtures`` with this checkout's interpreter. It +reports the CPython identities and ancestry of its assignment-flow examples. +Analyze that same module separately with ``source-ancestry`` to compare native +inference with the language runtime. This distinct command never imports an +application supplied through an ancestry request. + Making A Project ---------------- @@ -234,6 +553,68 @@ Note, however, that the use of ``automatic_soa`` is discouraged, because it may slow down saving considerably. +Function Parameter Information +------------------------------ + +For ordinary functions and methods, ``PyFunction.get_parameters()`` includes +positional-only and keyword-only bindings. A ``ParameterName.index`` is an +object slot, not a position in the source signature. The regular slots are +ordered as positional-only, positional-or-keyword, then keyword-only; +``*args`` and ``**kwargs`` bindings follow them. Signature separators do not +occupy slots. + +``get_param_names(special_args=False)`` returns the regular names in this +order. ``get_parameter_layout()`` returns a tuple of ``(kind, name)`` pairs +for all bindings, with kinds ``posonly``, ``positional``, ``kwonly``, +``vararg``, and ``kwarg``. Use ``get_positional_param_names()`` and +``get_keyword_param_names()`` when matching positional and keyword +arguments. In particular, a keyword with the same name as a positional-only +parameter can belong to ``**kwargs`` and is not a reference to that parameter. + +``get_parameter_defaults()`` maps names to their default-expression AST +nodes. Required parameters are absent from this mapping; an explicit +``=None`` has an AST node. Defaults belong to the definition's enclosing +scope. These APIs describe bindings and do not serialize a modified signature. +They do not provide a general annotation-expression scope model; existing +same-name annotation shadowing remains a separate limitation. + +Positional Change Signature operations, Introduce Parameter, Inline Parameter, +and Move Method raise ``RefactoringError`` for signatures containing positional-only +or keyword-only parameters before generating changes. Explicit keyword-only +``ArgumentAdder`` operations preserve modern headers and analyze every resource +the caller selected. Import-connected components classify internal provenance; +they never exclude selected consumers from validation or rewriting. +When a multiline header or call closes on its own line, additions retain that +layout and the existing parameter indentation. A missing separator is inserted +at the preceding argument's syntactic end, preserving its trailing comment; +new keywords before ``**kwargs`` retain their own lines as well. + +For disconnected source scopes, an unresolved declared callback parameter has no +target effect to prove; its arguments still require namespace-absence proof. +Unresolved attribute, subscription, factory and dynamic-import consumers remain +fail-closed across the full selected resource boundary. +Import-connected public scopes remain part of that boundary even without a +local invocation; an idle unresolved callback is not an effect contract. Native +integer operators are distinguished by the SDK's actual builtin binding when +analyzing numeric effects, while user-defined/external operators remain checked. +Required additions build the import-connected effect boundary. Defaulted +additions preserve existing call arity and omit unrequested keyword values, so +they retain native occurrence/decorator validation and compilation without an +inactive required-effect inference pass. +The single standard-library ``contextlib.contextmanager`` decorator is +authenticated by its native declaration and forwards the added keywords to +the original generator, including bound methods. Imported aliases retain that +identity; local homonyms and unproved decorator stacks remain refused. + +Use Function refuses keyword-only parameters. Existing positional-only Inline Method calls remain +supported, while keyword-only Inline Method calls are refused. Call inference +keeps unknown ``*args`` and ``**kwargs`` expansions conservative. + +Local to Field requires a named positional receiver. It refuses methods with +no positional parameters and static methods, while retaining instance-method +and class-method field conversions. + + Closing The Project ------------------- @@ -335,7 +716,7 @@ Retrieve the path relative to the project's root directory. .. code-block:: python # Get the path relative to the project's root directory. - relpath = libutils.relative(myproject.address, path) + relpath = libutils.path_relative_to_project_root(myproject, path) ``libutils.path_to_resource()`` @@ -427,6 +808,22 @@ class that provides this interface. ``rope.base.project.Project`` accepts an ``fscommands`` argument. You can use this argument to force rope to use your new class. +The same interface owns resource queries: ``exists``, ``isfile``, +``isdir``, ``listdir``, ``stat``, and ``realpath``. A captured filesystem can +therefore keep both reads and directory elections independent of later +disk changes. Resource observers obtain the original modification and size +indicator from this same ``stat`` port. Directory enumeration and metadata +errors propagate to the caller. + +Resources outside the project root are owned by ``NoProject``. +For a closed analysis, construct ``NoProject(fscommands=captured)`` and +pass it as ``external_project`` to ``Project`` together with the same +``fscommands``. This keeps the project's ordinary module lookup order +and lexical scopes while binding external reads to the captured +filesystem. It does not modify the process-wide ``get_no_project()`` +instance. Such read-only projects should use ``ropefolder=None``, +``save_objectdb=False``, and ``save_history=False``. + ``.ropeproject`` Folder ======================= @@ -456,6 +853,20 @@ Also ``Pycore.analyze_module()`` collects object information for a module. The collected information can be used to enhance rope's static object inference. +For an undecorated module function stored by a single class-body name +assignment, direct and saved instance accesses retain the receiver. Inherited +slots and imported functions keep the original function, class slot, and +parameter definitions, so field and keyword renames use those definitions. +Call analysis and saved object records also preserve the bound callable and +receiver across closing and reopening a project. + +This inference uses the existing ordinary argument protocol and requires a +positional parameter for the receiver, without positional-only or keyword-only +parameters. Recognized function writes to an instance, including aliases, +remain unbound. Override checks cover the storage, receiver, and access +modules; arbitrary writes in other modules, decorated storage classes, +metaclasses, and multiple storage assignments are outside this inference. + `rope.base.taskhandle.TaskHandle` ================================= @@ -533,6 +944,25 @@ representing this operation (which will be an instance of e.g. mover = create_move(project, resource, offset) +For a nested class, ``create_move`` returns ``MoveClass``. Its +``get_changes(destination_resource, destination_offset=None, resources=None)`` +transfers the declaration to a module or, when an offset is supplied, an existing +class. Native references and required imports move with the declaration; the +returned ``ChangeSet`` does not change resources until applied. Unresolved +references, occupied destinations and dependencies on an enclosing class are +rejected before publication. +An optional ``match_filter(occurrence)`` predicate can retain references whose +public access path is maintained by the caller's composed destination; direct +source references otherwise follow the native declaration identity. +Quoted type annotations use the native source-binding index, including nested +forward references. Literal values, annotation metadata, executable strings and +docstrings remain outside this semantic type-reference surface. +Package reexports, alias chains and inherited nested classes retain their native +binding identities; C3 chooses overrides rather than matching names. Explicit +namespace roots and class paths have no policy depth limit. Transfers into an +existing holder normalize self-imports through ImportUtils and preserve evaluated +sibling bases in the destination class's lexical scope. + Where ``resource`` and ``offset`` is the location to perform the refactoring. @@ -546,6 +976,16 @@ method: Where the ``destination`` module/package is the destination resource for move refactoring. Other refactorings classes have a similar interface. +``MoveMethod.get_class_changes`` plans transfers of static and class methods +to an existing source class. When the source and destination share a module, +their existing module bindings remain in place; the planner must not create a +self-import for those bindings. Function-local bindings are resolved through +the function scope's names, including assignments, while native identity and +collision checks still apply to dependencies outside that scope. +Cross-module moves select imports used by the moved fragment. A fragment that +declares export metadata retains that publication dependency set; unrelated +export metadata from the source module is not imported into a moved method. + List Of Refactorings -------------------- @@ -569,6 +1009,13 @@ folder. * ``rope.refactor.inline``: Inline occurrences of a method/variable/parameter. + Removing a variable definition can raise ``RefactoringError`` when it would + discard an unused initializer that is not known to be safe, remove a module + or class annotation, or leave reads outside the selected occurrences. + ``remove=False`` keeps the definition while replacing selected reads. + Selecting resources that exclude the defining module also keeps its + definition. Function-local annotations, safe literals, and proven bound + names remain eligible for removal. * ``rope.refactor.usefunction``: Try to use a function wherever possible. @@ -610,6 +1057,13 @@ them. If it is ``None`` all python modules in the project will be analyzed. Using this parameter, IDEs can let the user limit the files on which a refactoring should be applied. +When renaming a namespace package directory without an ``__init__.py``, +the default project-wide operation also moves the directory. With an +explicit resource list, the directory moves only when all of its Python +files are selected and the package has at least one Python file. Selecting +only importing modules updates those files while leaving the directory +in place, as with an ordinary package whose initializer is not selected. + Examples ======== @@ -617,6 +1071,18 @@ Examples Rename ------ +Rope follows a single unconditional assignment such as ``Holder.value = Item()`` +when both classes are defined at module level in the same module and the value +has ordinary data lookup behavior. Imported names in a class body retain their +original source identity when the corresponding class attribute is replaced. + +Repeated or conditional writes, early or deferred reads, class rebinding, +inheritance, and custom attribute lookup can leave the value uncertain. For +these modeled writes, a potentially affected rename raises +``rope.base.exceptions.RefactoringError`` before producing changes. This +analysis is conservative and may refuse an otherwise safe rename; it does not +infer arbitrary factories, descriptors, or dynamic cross-module assignments. + Using rename refactoring: .. code-block:: python @@ -817,6 +1283,13 @@ See pydocs and source code for more information (other functions in this module might be interesting, too; like ``get_doc``, ``get_definition_location``). +For PEP 695 type aliases, ``PyName.get_definition_location()`` retains the +declaring module and first binding line in module, function, and class scopes. +Imported aliases preserve that declaration location. Reading this metadata does +not evaluate the lazy alias value or substitute its right-hand side for the alias. +Aliases declared inside constructors and methods remain local bindings, not +instance attributes. + `rope.contrib.findit` --------------------- @@ -875,6 +1348,62 @@ It provides two new search methods: - search() - simpler wrapper around search_full with a basic sorting algorithm +Source binding environments +--------------------------- + +The structural scope of an expression and the environment in which its names +are resolved can differ. Function annotations use their declaration or +annotation environment; lambda parameters and comprehension targets have their +own bindings. The existing ``GlobalScope.get_inner_scope_for_offset(offset)`` +continues to return the structural scope. Its explicit +``get_structural_scope_for_offset(offset)`` alias has the same behavior. + +A module's ``get_source_binding_index()`` provides source identity queries: + +The public getter and direct ``SourceBindingIndex`` construction return a complete +index. Internal definition and expression queries share that same module index, +but may defer function bodies until their lexical source is queried. A full public +index request or public binding-scope enumeration completes those bodies before +returning, including nested annotations whose binding parent is the module. +Body demands include decorators on the body's first declaration. Parsing and native C3 +resolution are unchanged. Modules containing assignment expressions remain eager: +their nested headers can write bindings into enclosing or global scopes. + +.. code-block:: python + + pymodule = project.get_pymodule(resource) + index = pymodule.get_source_binding_index() + context = index.context_at(offset) + environment = context.environment + structural_scope = context.structural_scope + binding = index.binding_for_token(offset) + +``context_for_node(node)`` resolves a node from that module's AST. Source +queries in ``rope.base.evaluate`` use these environments while keeping source +identities separate from call-time values. A lambda's private argument matcher +handles its own positional, keyword and creation-time default values; it does +not change the ordinary ``Arguments`` or ``PyFunction`` parameter protocols. +Unknown argument expansion or conflicting values remain unknown. + +When a directly assigned class attribute and a directly assigned instance +attribute in ``__init__`` hold different lambdas, attribute value views select +the class or initialized instance value independently. This selection does not +replace their source names or change the ordinary class attribute protocol. + +Ordinary positional-only and keyword-only parameters are represented by +distinct readable source names, so their bodies do not fall back to same-named +outer variables. Their call inference and signature rewriting are outside this +model's supported range. Rename refuses edits to these parameter bindings, and +introduce-parameter refuses to rewrite such a signature. Queries still return +definition locations and an unknown value. An unresolved generic annotation +binding likewise refuses a potentially affected edit before any source changes. + +The model does not provide arbitrary dynamic class namespace inference. For +example, assigning a lambda to ``Owner.callback`` after the class definition +can leave attribute inference unknown; source binding identity is not a promise +of complete dynamic value inference. Inline safety for deferred annotations and +initializer removal is outside the source model's new consumer scope. + Cross-Project Refactorings -------------------------- @@ -931,3 +1460,20 @@ calling ``project.do(changes)`` for each tuple, or use project_and_changes = renamer.get_all_changes('newname') multiproject.perform(project_and_changes) + + +Unchanged source value descriptors +--------------------------------- + +Native attribute inference preserves the identity of a value returned unchanged +by a source descriptor. This applies when the source constructor stores its +argument directly and ``__get__`` returns that stored attribute, for both class +and instance access. Inherited exports and keyword constructor arguments retain +the same identity. Descriptors whose constructor or getter transforms the value +do not acquire this identity proof. + +The existing override guard consumes the source binding index's attribute-write +inventory. That lazy inventory belongs to one immutable parsed module snapshot; +recapturing a changed resource creates a new index rather than reusing previous +write observations. Declaration identities and refactoring safety remain owned +by the native binding and occurrence engines. diff --git a/docs/overview.rst b/docs/overview.rst index 01076ade5..79036fb2d 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -400,7 +400,7 @@ Extract Method In these examples ``${region_start}`` and ``${region_end}`` show the selected region for extraction: -.. code-block:: none +.. code-block:: python def a_func(): a = 1 @@ -421,7 +421,7 @@ After performing extract method we'll have: For multi-line extractions if we have: -.. code-block:: none +.. code-block:: python def a_func(): a = 1 @@ -1265,7 +1265,7 @@ After running ``mod1`` either SOA or DOA on this module you can test: ``mod2.py``: -.. code-block:: none +.. code-block:: python import mod1 @@ -1407,6 +1407,15 @@ Nested expressions also allowed: - collections.Iterable[list[Foo]] +Union alternatives: + +- Foo | Bar + +Attribute completion includes the attributes exposed by each resolved +alternative. If an alternative cannot be resolved, the complete union remains +unresolved. A resolved ``types.NoneType`` remains an alternative; it is distinct +from the resolver's absent-result sentinel. + TODO: Callable objects: diff --git a/docs/release-process.rst b/docs/release-process.rst index 87c356864..45af9819a 100644 --- a/docs/release-process.rst +++ b/docs/release-process.rst @@ -20,6 +20,44 @@ Release 7. Create Github Release 8. Publish release announcements to GitHub Discussions +Downstream Fork Releases +------------------------ + +Local-version releases (a ``+`` label) are distributed as GitHub Release wheel and +source-distribution assets, not uploaded to PyPI. The `version specification`_ +prohibits local version identifiers on PyPI. Public upstream releases retain the +existing trusted-publisher workflow. + +Run ``make setup fmt check test build`` on the release candidate. The native +``build`` target invokes ``artifact-check`` to verify typed markers and exercise +the installed wheel outside the checkout. After merging and creating the signed, +annotated version tag, create the GitHub Release with both validated artifacts. +The artifact check prints the SHA-256 of each built distribution so a development +candidate can be reviewed by content identity before installation or publication. +The selected artifact version comes from the producer's ``pyproject.toml``, +never from a previously installed distribution in the build environment. +The artifact checker reads that manifest with ``toml``, declared in the development +extra as well as the release extra. Both the producer and the isolated +``wheel[dev]`` consumer install it. +The release workflow downloads those published assets and runs the same +``make artifact-check`` against them; a locally rebuilt wheel is not a substitute +for that published-artifact receipt. + +To repeat validation without changing an existing tag or its assets, dispatch the +maintained controller with ``gh workflow run python-publish.yml --ref dc-use +-f release_tag= -f release_commit=``. In this downstream +fork, ``dc-use`` owns the maintained release controller; use the maintained +controller branch when applying this flow in another repository. The recorded full +producer commit pins the checkout, and the job rejects a tag pointing elsewhere. +It reports downloaded asset hashes and validates the published distributions, +rather than building replacements. Setup uses the installed +``uv`` executable: invoking ``python -m uv`` while Make exports a bytecode cache +inside the future virtualenv would create that directory before its safety check. +Native interpreter discovery uses ``uv python find --no-project`` so an ambient +workspace selection cannot impose an unrelated project's Python constraint. + +.. _version specification: https://packaging.python.org/en/latest/specifications/version-specifiers/#local-version-identifiers + Release Schedule ================ diff --git a/pyproject.toml b/pyproject.toml index 062ce663a..4dcb716cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = 'rope' description = 'a python refactoring library...' readme = 'README.rst' -requires-python = '>=3.10' +requires-python = '>=3.13' classifiers = [ 'Development Status :: 4 - Beta', 'Operating System :: OS Independent', @@ -10,19 +10,17 @@ classifiers = [ 'Environment :: Win32 (MS Windows)', 'Environment :: MacOS X', 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)', 'Natural Language :: English', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3.14', 'Programming Language :: Python :: 3.15', 'Topic :: Software Development', ] -version = '1.15.1' +version = '1.15.2+dc.8' +license = 'LGPL-3.0-or-later' +license-files = ['COPYING'] dependencies = ['pytoolconfig[global] >= 1.2.2'] [[project.authors]] @@ -33,9 +31,6 @@ email = 'aligrudi@users.sourceforge.net' name = 'Lie Ryan' email = 'lieryan.24@proton.me' -[project.license] -text = 'LGPL-3.0-or-later' - [project.urls] Source = 'https://github.com/python-rope/rope' Documentation = 'https://rope.readthedocs.io/' @@ -51,37 +46,25 @@ dev = [ 'pytest>=7.0.1', 'pytest-cov>=4.1.0', 'pytest-timeout>=2.1.0', + 'pytest-xdist>=3.8.0', 'build>=0.7.0', - 'pre-commit>=2.20.0', + 'pre-commit>=3.2.0', + 'toml>=0.10.2', ] release = [ 'toml>=0.10.2', 'twine>=4.0.2', 'pip-tools>=6.12.1', ] -[tool.setuptools] -packages = [ - 'rope', - 'rope.base', - 'rope.base.oi', - 'rope.base.oi.type_hinting', - 'rope.base.oi.type_hinting.providers', - 'rope.base.oi.type_hinting.resolvers', - 'rope.base.utils', - 'rope.contrib', - 'rope.contrib.autoimport', - 'rope.refactor', - 'rope.refactor.importutils', -] +[tool.setuptools.packages.find] +namespaces = false +exclude = ['ropetest*'] + +[tool.setuptools.package-data] +rope = ["py.typed"] [tool.black] -target-version = [ - 'py38', - 'py39', - 'py310', - 'py311', - 'py312', -] +target-version = ['py313'] include = 'rope/.*\.pyi?$' force-exclude = 'ropetest|rope/base/prefs.py' @@ -105,13 +88,13 @@ python_files = [ "*test.py", "__init__.py", ] - -markers = [ - "time_limit: sets a maximum amount of time the test can run", -] +filterwarnings = ["error"] [build-system] requires = [ - 'setuptools', + 'setuptools>=77.0.0', ] build-backend = 'setuptools.build_meta' + +[tool.coverage.run] +source = ['rope', 'ropetest'] diff --git a/rope/base/arguments.py b/rope/base/arguments.py index 12a626816..be77d81ff 100644 --- a/rope/base/arguments.py +++ b/rope/base/arguments.py @@ -10,9 +10,10 @@ class Arguments: """ - def __init__(self, args, scope): + def __init__(self, args, scope, pyfunction=None): self.args = args self.scope = scope + self.pyfunction = pyfunction self.instance = None def get_arguments(self, parameters): @@ -25,6 +26,8 @@ def get_arguments(self, parameters): return result def get_pynames(self, parameters): + if isinstance(self.pyfunction, rope.base.pyobjects.PyFunction): + return self._get_function_pynames(parameters) result = [None] * max(len(parameters), len(self.args)) for index, arg in enumerate(self.args): if isinstance(arg, ast.keyword) and arg.arg in parameters: @@ -33,6 +36,44 @@ def get_pynames(self, parameters): result[index] = self._evaluate(arg) return result + def _get_function_pynames(self, parameters): + positional = [ + name + for name in self.pyfunction.get_positional_param_names() + if name in parameters + ] + keywords = set(self.pyfunction.get_keyword_param_names()) & set(parameters) + result = dict.fromkeys(parameters) + provided = set() + position = 0 + unknown_positions = False + unknown_keywords = False + for arg in self.args: + if isinstance(arg, ast.Starred): + unknown_positions = True + elif isinstance(arg, ast.keyword): + if arg.arg is None: + unknown_keywords = True + elif arg.arg in keywords: + result[arg.arg] = self._evaluate(arg.value) + provided.add(arg.arg) + elif not unknown_positions: + if position < len(positional): + result[positional[position]] = self._evaluate(arg) + provided.add(positional[position]) + position += 1 + for name, default in self.pyfunction.get_parameter_defaults().items(): + if name not in result or name in provided: + continue + if unknown_positions and name in positional: + continue + if unknown_keywords and name in keywords: + continue + result[name] = rope.base.evaluate.eval_node( + self.pyfunction.parent.get_scope(), default + ) + return [result[name] for name in parameters] + def get_instance_pyname(self): if self.args: return self._evaluate(self.args[0]) @@ -41,15 +82,55 @@ def _evaluate(self, ast_node): return rope.base.evaluate.eval_node(self.scope, ast_node) -def create_arguments(primary, pyfunction, call_node, scope): +def create_arguments(primary, pyfunction, call_node, scope, ignore_instance=False): """A factory for creating `Arguments`""" args = list(call_node.args) args.extend(call_node.keywords) called = call_node.func - # XXX: Handle constructors - if _is_method_call(primary, pyfunction) and isinstance(called, ast.Attribute): - args.insert(0, called.value) - return Arguments(args, scope) + result = Arguments(args, scope, pyfunction) + if ignore_instance or not isinstance(called, ast.Attribute): + return result + return bind_method(primary, pyfunction, result, scope) + + +def bind_method(primary, pyfunction, args, scope, owner=None): + """Bind an attribute or special method through the native descriptor kind. + + ``owner`` is the class supplying a special-method descriptor. Ordinary + attribute calls derive the receiving class from ``primary`` instead. + """ + from rope.base.sourcebindings import SourceLambdaFunction + + if primary is None: + return args + bound = rope.base.builtins.bound_receiver(primary.get_object()) + if bound is not primary.get_object(): + primary = rope.base.pynames.UnboundName(bound) + if isinstance(pyfunction, SourceLambdaFunction): + if pyfunction.bound_receiver is not None: + return args + kind = pyfunction.descriptor_kind + elif isinstance(pyfunction, rope.base.pyobjects.PyFunction): + kind = pyfunction.get_kind() + else: + kind = None + if kind is not None: + if kind == "staticmethod": + return args + receiver = primary.get_object() + if kind == "classmethod": + if owner is not None: + receiver = owner + elif not isinstance(receiver, rope.base.pyobjects.AbstractClass): + receiver = receiver.get_type() + return MixedArguments(rope.base.pynames.UnboundName(receiver), args, scope) + if owner is not None or ( + kind == "method" and _is_method_call(primary, pyfunction) + ): + return MixedArguments(primary, args, scope) + elif owner is not None or _is_method_call(primary, pyfunction): + return MixedArguments(primary, args, scope) + return args class ObjectArguments: @@ -79,6 +160,13 @@ def __init__(self, pyname, arguments, scope): self.args = arguments def get_pynames(self, parameters): + if not parameters: + return [] + function = getattr(self.args, "pyfunction", None) + if isinstance(function, rope.base.pyobjects.PyFunction): + positional = function.get_positional_param_names() + if not positional or parameters[0] != positional[0]: + return self.args.get_pynames(parameters) return [self.pyname] + self.args.get_pynames(parameters[1:]) def get_arguments(self, parameters): diff --git a/rope/base/ast.py b/rope/base/ast.py index cc376e0bc..b090eed5d 100644 --- a/rope/base/ast.py +++ b/rope/base/ast.py @@ -8,18 +8,31 @@ # Suppress the mypy complaint: Module "ast" has no attribute "_const_node_type_names" from ast import _const_node_type_names # type: ignore except ImportError: - # backported from stdlib `ast` - assert sys.version_info < (3, 8) or sys.version_info >= (3, 14) - _const_node_type_names = { - bool: "NameConstant", # should be before int - type(None): "NameConstant", - int: "Num", - float: "Num", - complex: "Num", - str: "Str", - bytes: "Bytes", - type(...): "Ellipsis", - } + if sys.version_info >= (3, 8): + # Python 3.14+ removed _const_node_type_names; all constant literals + # are represented as ast.Constant nodes. + _const_node_type_names = { + bool: "Constant", # should be before int + type(None): "Constant", + int: "Constant", + float: "Constant", + complex: "Constant", + str: "Constant", + bytes: "Constant", + type(...): "Constant", + } + else: + # Python < 3.8 used separate node types (Num, Str, Bytes, etc.) + _const_node_type_names = { + bool: "NameConstant", # should be before int + type(None): "NameConstant", + int: "Num", + float: "Num", + complex: "Num", + str: "Str", + bytes: "Bytes", + type(...): "Ellipsis", + } def parse(source, filename="", *args, **kwargs): # type: ignore diff --git a/rope/base/builtins.py b/rope/base/builtins.py index 9c8bd1611..21eda74c8 100644 --- a/rope/base/builtins.py +++ b/rope/base/builtins.py @@ -1,14 +1,21 @@ """This module tries to support builtin types and functions.""" +import collections.abc import inspect import io +import sys +import types +import typing +from functools import partial import rope.base.evaluate from rope.base import arguments, ast, pynames, pyobjects, utils class BuiltinModule(pyobjects.AbstractModule): - def __init__(self, name, pycore=None, initial={}): + def __init__(self, name, pycore=None, initial=None): + if initial is None: + initial = {} super().__init__() self.name = name self.pycore = pycore @@ -26,6 +33,14 @@ def get_doc(self): def get_name(self): return self.name.split(".")[-1] + @utils.saveit + def get_scope(self): + # Module scope over the native attribute mapping; builtin modules + # carry no source, so the usual parsed-module scope does not apply. + import rope.base.pyscopes + + return rope.base.pyscopes.BuiltinModuleScope(self) + @property @utils.saveit def attributes(self): @@ -78,17 +93,324 @@ def __init__(self, builtin, attributes, parent=None): @utils.saveit def get_attributes(self): result = _object_attributes(self.builtin, self) + typing_alias = any( + self.builtin is owner for owner in (typing.Generic, typing.Protocol) + ) + if typing_alias or ( + inspect.isclass(self.builtin) + and self.builtin.__module__ == "builtins" + and hasattr(self.builtin, "__class_getitem__") + ): + parent = self.parent + project = ( + parent.pycore.project + if isinstance(parent, BuiltinModule) and parent.pycore + else None + ) + result["__class_getitem__"] = _class_getitem(self, project, typing_alias) result.update(self.initial) return result + @utils.saveit + def get_superclasses(self): + if not inspect.isclass(self.builtin): + return [] + return [_runtime_class(base) for base in self.builtin.__bases__] + + @utils.saveit + def get_mro(self): + if not inspect.isclass(self.builtin): + return [self] + return [self] + [_runtime_class(cls) for cls in self.builtin.__mro__[1:]] + + def get_consistent_mro(self): + return self.get_mro() + + def _get_own_attributes(self): + if not inspect.isclass(self.builtin): + return self.get_attributes() + own = vars(self.builtin) + return { + name: pyname + for name, pyname in self.get_attributes().items() + if name in own or name in self.initial + } + def get_module(self): return builtins + def get_metaclass(self): + return _runtime_class(type(self.builtin)) + + +_runtime_classes = {} + + +def _runtime_class(cls): + """Return the rope class standing for the runtime class `cls` + + A class of the ``builtins`` module is the object the builtins module + exposes, so it is the same object a source class names as a base. + + """ + if cls.__module__ == "builtins": + pyname = builtins.get_attributes().get(cls.__name__) + if pyname is not None: + pyobject = pyname.get_object() + if isinstance(pyobject, BuiltinClass) and pyobject.builtin is cls: + return pyobject + if cls not in _runtime_classes: + _runtime_classes[cls] = BuiltinClass(cls, {}) + return _runtime_classes[cls] + + +def _is_runtime_class(value, runtime_class, *, binding=None, attribute=None): + """Authenticate a native class against its runtime library declaration.""" + if isinstance(value, BuiltinClass): + return value.builtin is runtime_class + if isinstance(binding, pynames.ImportedName): + module = binding.imported_module.get_object() + attribute = binding.imported_name + elif isinstance(binding, pynames.ImportedModule): + module = binding.get_object() + else: + module = None + if isinstance(module, pyobjects.PyModule) and attribute is not None: + resource = module.get_resource() + name = module.get_name() + observed = sys.modules.get(name) + if ( + name.partition(".")[0] in sys.stdlib_module_names + and isinstance(observed, types.ModuleType) + and resource is not None + and isinstance(vars(observed).get("__file__"), str) + and utils.same_file_path(resource.real_path, vars(observed)["__file__"]) + and vars(observed).get(attribute) is runtime_class + ): + return True + if not isinstance(value, pyobjects.PyClass): + return False + module = value.get_module() + resource = module.get_resource() + runtime_module = inspect.getmodule(runtime_class) + return ( + value.parent is module + and value.get_name() == runtime_class.__name__ + and resource is not None + and runtime_module is not None + and utils.same_file_path(resource.real_path, runtime_module.__file__) + ) + + +def _is_runtime_form(value, runtime_form): + """Authenticate a typing special form against its runtime declaration. + + ``typing.Unpack`` is a ``_SpecialForm``-decorated function, not a class, + so class authentication does not apply: builtin-side values match by + identity, source-side functions by their declaring module and name. + """ + if isinstance(value, (BuiltinFunction, BuiltinUnknown)): + return value.builtin is runtime_form + if not isinstance(value, pyobjects.PyFunction): + return False + module = value.get_module() + resource = module.get_resource() + runtime_module = inspect.getmodule(runtime_form) + return ( + value.parent is module + and value.get_name() == getattr(runtime_form, "__name__", None) + and resource is not None + and runtime_module is not None + and utils.same_file_path(resource.real_path, runtime_module.__file__) + ) + + +def _runtime_unpack(value): + """Return the runtime ``Unpack`` form `value` stands for, if any.""" + for form in ( + getattr(typing, "Unpack", None), + getattr(sys.modules.get("typing_extensions"), "Unpack", None), + ): + if form is not None and _is_runtime_form(value, form): + return form + return None + + +def _class_getitem(owner, project, typing_alias): + return BuiltinName( + BuiltinFunction( + function=partial( + _generic_alias, project=project, typing_alias=typing_alias + ), + argnames=["cls", "key"], + parent=owner, + ) + ) + + +def _generic_alias(context, project, typing_alias): + origin = context.get_argument("cls") + key = context.get_argument("key") + if origin is None or key is None: + # A subscript whose origin or key could not be inferred yields no + # alias and raises nothing; callers handle the absence. + return None + arguments_ = ( + key.get_type().get_holding_objects() + if key is not None and isinstance(key.get_type(), Tuple) + else (key,) + ) + if any( + _is_runtime_class(origin, owner) for owner in (typing.Generic, typing.Protocol) + ): + variables = tuple( + typing.__dict__[name] + for name in ("TypeVar", "ParamSpec", "TypeVarTuple") + if name in typing.__dict__ + ) + direct = tuple( + typing.__dict__[name] + for name in ("TypeVar", "ParamSpec") + if name in typing.__dict__ + ) + unwrapped = [] + invalid = None + for argument in arguments_: + # PEP 646: a TypeVarTuple parameter enters only unpacked — the + # ``*Ts`` subscription marker or an explicit ``Unpack[Ts]`` + # alias; a raw ``Cls[Ts]`` subscript is the TypeError the + # runtime raises. + if isinstance(argument, UnpackedTypeVarTuple): + allowed = variables + argument = argument.target + elif ( + isinstance(argument, GenericAlias) + and argument.typing_alias + and len(argument.arguments) == 1 + and _runtime_unpack(argument.origin) is not None + ): + allowed = variables + argument = argument.arguments[0] + else: + allowed = direct + if argument is None or not any( + _is_runtime_class(argument.get_type(), variable) for variable in allowed + ): + invalid = argument + break + unwrapped.append(argument) + if invalid is not None or not unwrapped: + error = TypeError(f"{origin.get_name()} parameters must be type variables") + notes = [ + ( + f"alias origin={type(origin).__name__}@{id(origin):x} " + f"key={type(key).__name__}@{id(key):x}" + ) + ] + try: + for index, argument in enumerate(arguments_): + if argument is None: + notes.append(f"argument[{index}]=None") + continue + for label, value in ( + ("object", argument), + ("type", argument.get_type()), + ): + module = value.get_module() + resource = module.get_resource() if module is not None else None + module_name = module.get_name() if module is not None else None + resource_path = ( + resource.real_path if resource is not None else None + ) + name = ( + value.get_name() + if isinstance(value, pyobjects.AbstractClass) + else None + ) + notes.append( + f"argument[{index}].{label}=" + f"{type(value).__name__}@{id(value):x} name={name!r} " + f"module={module_name!r} resource={resource_path!r}" + ) + except Exception as diagnostic_error: + notes.append(f"generic argument context failed: {diagnostic_error!r}") + error.__dict__["__notes__"] = notes + raise error + if len({id(argument) for argument in unwrapped}) != len(unwrapped): + raise TypeError(f"{origin.get_name()} parameters must be unique") + # The alias keeps the arguments as subscribed, like the runtime's + # ``__args__``; unwrapping is validation-internal. + return GenericAlias(origin, arguments_, project, typing_alias) + + +class GenericAlias(pyobjects.PyObject): + """The native value of a standard-library class subscription.""" + + def __init__(self, origin, arguments_, project, typing_alias): + super().__init__( + _runtime_class(typing._GenericAlias if typing_alias else types.GenericAlias) + ) + self.origin = origin + self.arguments = arguments_ + self.project = project + self.typing_alias = typing_alias + + def __eq__(self, other): + if not isinstance(other, GenericAlias): + return NotImplemented + return ( + self.origin is other.origin + and len(self.arguments) == len(other.arguments) + and all( + left is right for left, right in zip(self.arguments, other.arguments) + ) + ) + + def __ne__(self, other): + equal = self.__eq__(other) + return NotImplemented if equal is NotImplemented else not equal + + def __hash__(self): + return hash( + (id(self.origin), tuple(id(argument) for argument in self.arguments)) + ) + + def get_attributes(self): + return { + "__origin__": BuiltinName(self.origin), + "__args__": BuiltinName(get_tuple(*self.arguments)), + "__mro_entries__": BuiltinName( + BuiltinFunction(function=self._mro_entries, argnames=["self", "bases"]) + ), + } + + def _mro_entries(self, context): + original = context.get_argument("bases").get_type().get_holding_objects() + if _is_runtime_class(self.origin, typing.Generic): + protocol = self.project.get_module("typing")["Protocol"].get_object() + if any(base is protocol for base in original): + return get_tuple() + # Alias equality is origin plus arguments: an instance rebuilt + # from the same signature is not found by identity, and a + # missing alias must not leak StopIteration. + position = next( + (index for index, base in enumerate(original) if base == self), + None, + ) + if position is not None and any( + isinstance(base, GenericAlias) and base.typing_alias and base != self + for base in original[position + 1 :] + ): + return get_tuple() + return get_tuple(self.origin) + class BuiltinFunction(_BuiltinElement, pyobjects.AbstractFunction): def __init__( - self, returned=None, function=None, builtin=None, argnames=[], parent=None + self, returned=None, function=None, builtin=None, argnames=None, parent=None ): + if argnames is None: + argnames = [] _BuiltinElement.__init__(self, builtin, parent) pyobjects.AbstractFunction.__init__(self) self.argnames = argnames @@ -145,9 +467,15 @@ def _create_builtin_type_getter(cls): def _get_builtin(*args): if not hasattr(cls, "_generated"): cls._generated = {} - if args not in cls._generated: - cls._generated[args] = cls(*args) - return cls._generated[args] + # Identity-keyed: plain ``PyObject``s hash and compare by type, + # so a value-keyed cache collapses distinct inferred objects + # (each module's own ``TypeVar``/``TypeVarTuple``) onto one + # cached builtin. The cached instance holds its arguments + # strongly, so an id cannot be reused while its entry lives. + key = tuple(id(arg) for arg in args) + if key not in cls._generated: + cls._generated[key] = cls(*args) + return cls._generated[key] return _get_builtin @@ -167,6 +495,11 @@ def __init__(self, argnames, args): self.args = args def _get_scope_and_pyname(self, pyname): + # A value overlay keeps the source name its per-name data belongs to. + from rope.base.sourcebindings import SourceValueName + + if isinstance(pyname, SourceValueName): + pyname = pyname.source_pyname if pyname is not None and isinstance(pyname, pynames.AssignedName): pymodule, lineno = pyname.get_definition_location() if pymodule is None: @@ -221,7 +554,11 @@ def save_per_name(self, value): scope, name = self._get_scope_and_pyname(pyname) if name is not None: pymodule = pyname.get_definition_location()[0] - pymodule.pycore.object_info.save_per_name(scope, name, value) + # Per-name container state records element types, not positions, + # so a value sampled from another container keeps its type. + pymodule.pycore.object_info.save_per_name( + scope, name, pyobjects.unsampled(value) + ) class _AttributeCollector: @@ -234,10 +571,12 @@ def __call__( name, returned=None, function=None, - argnames=["self"], + argnames=None, check_existence=True, parent=None, ): + if argnames is None: + argnames = ["self"] try: builtin = getattr(self.type, name) except AttributeError: @@ -258,6 +597,55 @@ def __setitem__(self, name, value): self.attributes[name] = value +def _read_mutable_holding(value, origin): + return pyobjects.sampled_value(value, origin) + + +class Sequence(BuiltinClass): + """A read-only native sequence retaining its element object.""" + + def __init__(self, holding=None): + self.holding = holding + collector = _AttributeCollector(collections.abc.Sequence) + collector( + "__getitem__", function=self._item, argnames=["self", "key"], parent=self + ) + for name in ("__iter__", "__reversed__"): + collector(name, returned=get_iterator(holding), parent=self) + for name in ("__len__", "index", "count"): + collector( + name, + returned=pyobjects.PyObject(builtins["int"].get_object()), + parent=self, + ) + collector( + "__contains__", + returned=pyobjects.PyObject(builtins["bool"].get_object()), + parent=self, + ) + super().__init__(collections.abc.Sequence, collector.attributes) + + def get_holding_objects(self): + return (self.holding,) + + def _item(self, context): + key = context.get_argument("key") + if key is None: + return pyobjects.get_unknown() + if key.get_type() == builtins["slice"].get_object(): + return get_sequence(self.holding) + if key.get_type() in ( + builtins["int"].get_object(), + builtins["bool"].get_object(), + ): + return self.holding + return pyobjects.get_unknown() + + +get_sequence = _create_builtin_getter(Sequence) +get_sequence_type = _create_builtin_type_getter(Sequence) + + class List(BuiltinClass): def __init__(self, holding=None): self.holding = holding @@ -299,6 +687,9 @@ def __init__(self, holding=None): super().__init__(list, collector.attributes) + def get_holding_objects(self): + return (self.holding,) + def _new_list(self, args): return _create_builtin(args, get_list) @@ -318,16 +709,18 @@ def _self_set(self, context): context.save_per_name(holding) def _list_get(self, context): - if self.holding is not None: - args = context.get_arguments(["self", "key"]) - if ( - len(args) > 1 - and args[1] is not None - and args[1].get_type() == builtins["slice"].get_object() - ): - return get_list(self.holding) - return self.holding - return context.get_per_name() + holding = _read_mutable_holding( + self.holding if self.holding is not None else context.get_per_name(), self + ) + args = context.get_arguments(["self", "key"]) + if ( + args is not None + and len(args) > 1 + and args[1] is not None + and args[1].get_type() == builtins["slice"].get_object() + ): + return get_list(holding) + return holding def _iterator_get(self, context): return get_iterator(self._list_get(context)) @@ -340,6 +733,98 @@ def _self_get(self, context): get_list_type = _create_builtin_type_getter(List) +class LiteralString(pyobjects.PyObject): + """A known string value retaining the canonical builtin string class.""" + + def __init__(self, value): + super().__init__(get_str_type()) + self.value = value + + def __eq__(self, obj): + # Literal values are value objects: two known strings with the same + # text are the same object, unlike reference-like builtin types. + if isinstance(obj, LiteralString): + return self.value == obj.value + return NotImplemented + + def __ne__(self, obj): + result = self.__eq__(obj) + if result is NotImplemented: + return result + return not result + + def __hash__(self): + return hash((LiteralString, self.value)) + + +class UnpackedTypeVarTuple(pyobjects.PyObject): + """PEP 646 ``*Ts`` subscription marker holding the parameter target. + + The starred subscription syntax exists only at the expression level; + this marker carries the unpack fact from the evaluator into the typing + alias validation, which accepts a TypeVarTuple parameter only through + it (a raw ``Cls[Ts]`` subscript is the runtime ``TypeError``). + """ + + def __init__(self, target): + super().__init__(pyobjects.get_base_type("Unknown")) + self.target = target + + +class LiteralDict(pyobjects.PyObject): + """A literal namespace whose exact entries remain known until mutation.""" + + def __init__(self, keys, values, entries): + super().__init__(Dict(keys, values)) + self.entries = entries + result = self.type.get_attributes() + if entries is not None: + result["__getitem__"] = BuiltinName( + BuiltinFunction( + function=self._lookup, + argnames=["self", "key"], + builtin=dict.__getitem__, + ) + ) + for name in ( + "__setitem__", + "__delitem__", + "clear", + "pop", + "popitem", + "setdefault", + "update", + "__ior__", + ): + if name in result: + function = result[name].get_object() + result[name] = BuiltinName( + BuiltinFunction( + function=partial(self._mutate, function=function), + argnames=function.get_param_names(), + ) + ) + + def get_holding_objects(self): + if self.entries is None: + return (self.type,) + return (self.type, *(entry.get_object() for entry in self.entries.values())) + + def _lookup(self, context): + key = context.get_argument("key") + if ( + self.entries is not None + and isinstance(key, LiteralString) + and key.value in self.entries + ): + return _read_mutable_holding(self.entries[key.value].get_object(), self) + return pyobjects.get_unknown() + + def _mutate(self, context, function): + self.entries = None + return function.get_returned_object(context.args) + + class Dict(BuiltinClass): def __init__(self, keys=None, values=None): self.keys = keys @@ -359,7 +844,21 @@ def __init__(self, keys=None, values=None): collector("update", function=self._self_set, parent=self) super().__init__(dict, collector.attributes) + def get_holding_objects(self): + return (self.keys, self.values) + def _new_dict(self, args): + supplied = args.get_pynames(["sequence"]) + items = ( + pyobjects.unsampled(supplied[0].get_object()) + if supplied and supplied[0] is not None + else None + ) + if isinstance(items, NamespaceItems): + # Namespace pairs describe the element type; the sampled position + # of the read that produced them is irrelevant to their entries. + return items.namespace + def do_create(holding=None): if holding is None: return get_dict() @@ -377,12 +876,12 @@ def _dict_add(self, context): context.save_per_name(get_tuple(key, value)) def _item_get(self, context): - if self.keys is not None: - return get_tuple(self.keys, self.values) - item = context.get_per_name() + item = context.get_per_name() if self.keys is None else None if item is None or not isinstance(item.get_type(), Tuple): - return get_tuple(self.keys, self.values) - return item + holdings = (self.keys, self.values) + else: + holdings = item.get_type().get_holding_objects() + return get_tuple(*(_read_mutable_holding(value, self) for value in holdings)) def _value_get(self, context): item = self._item_get(context).get_type() @@ -443,8 +942,8 @@ def __init__(self, *objects): first = objects[0] attributes = { "__getitem__": BuiltinName( - BuiltinFunction(first) - ), # TODO: add slice support + BuiltinFunction(function=self._item, argnames=["self", "key"]) + ), "__getslice__": BuiltinName(BuiltinFunction(pyobjects.PyObject(self))), "__new__": BuiltinName(BuiltinFunction(function=self._new_tuple)), "__iter__": BuiltinName(BuiltinFunction(get_iterator(first))), @@ -454,6 +953,25 @@ def __init__(self, *objects): def get_holding_objects(self): return self.objects + def get_item(self, index): + if -len(self.objects) <= index < len(self.objects): + return self.objects[index] + + def _item(self, context): + key = context.get_argument("key") + if ( + key is not None + and key.get_type() + in ( + builtins["int"].get_object(), + builtins["bool"].get_object(), + ) + and self.objects + and all(value == self.objects[0] for value in self.objects) + ): + return self.objects[0] + return pyobjects.get_unknown() + def _new_tuple(self, args): return _create_builtin(args, get_tuple) @@ -487,6 +1005,9 @@ def __init__(self, holding=None): collector("__iter__", function=self._iterator_get, parent=self) super().__init__(set, collector.attributes) + def get_holding_objects(self): + return (self.holding,) + def _new_set(self, args): return _create_builtin(args, get_set) @@ -506,9 +1027,9 @@ def _self_set(self, context): context.save_per_name(holding) def _set_get(self, context): - if self.holding is not None: - return self.holding - return context.get_per_name() + return _read_mutable_holding( + self.holding if self.holding is not None else context.get_per_name(), self + ) def _iterator_get(self, context): return get_iterator(self._set_get(context)) @@ -584,17 +1105,21 @@ def get_definition_location(self): class Iterator(pyobjects.AbstractClass): def __init__(self, holding=None): super().__init__() + # The element type is proven; only the cursor position of a read is not. self.holding = holding self.attributes = { - "next": BuiltinName(BuiltinFunction(self.holding)), + "next": BuiltinName(BuiltinFunction(_read_mutable_holding(holding, self))), "__iter__": BuiltinName(BuiltinFunction(self)), } + def get_holding_objects(self): + return (self.holding,) + def get_attributes(self): return self.attributes def get_returned_object(self, args): - return self.holding + return _read_mutable_holding(self.holding, self) get_iterator = _create_builtin_getter(Iterator) @@ -605,13 +1130,16 @@ def __init__(self, holding=None): super().__init__() self.holding = holding self.attributes = { - "next": BuiltinName(BuiltinFunction(self.holding)), - "__iter__": BuiltinName(BuiltinFunction(get_iterator(self.holding))), + "next": BuiltinName(BuiltinFunction(_read_mutable_holding(holding, self))), + "__iter__": BuiltinName(BuiltinFunction(get_iterator(holding))), "close": BuiltinName(BuiltinFunction()), "send": BuiltinName(BuiltinFunction()), "throw": BuiltinName(BuiltinFunction()), } + def get_holding_objects(self): + return (self.holding,) + def get_attributes(self): return self.attributes @@ -727,14 +1255,239 @@ def parent(self): return self.scope.pyobject +def _attribute_access(context, *, class_access): + receiver = context.get_pyname("self") + name = context.get_argument("name") + if receiver is None or not isinstance(name, LiteralString): + return pyobjects.get_unknown() + value = receiver.get_object() + if ( + value is None + or value == pyobjects.get_unknown() + or isinstance(value, pyobjects.AbstractClass) != class_access + ): + return pyobjects.get_unknown() + try: + selected = value[name.value].get_object() + except rope.base.exceptions.AttributeNotFoundError: + return pyobjects.get_unknown() + if isinstance(selected, pyobjects.PyFunction): + for decorator in selected.decorators: + binding = rope.base.evaluate.eval_node( + selected.parent.get_scope(), decorator + ) + if binding is None or getattr( + binding.get_object(), "builtin", None + ) not in (staticmethod, classmethod): + return pyobjects.get_unknown() + kind = selected.get_kind() + if kind == "classmethod" or kind == "method" and not class_access: + from rope.base.classfunctions import BoundFunction + + if kind == "classmethod" and not class_access: + receiver = pynames.UnboundName(value.get_type()) + return BoundFunction(selected, receiver) + return selected + + +def _attribute_access_name(owner, *, class_access): + return BuiltinName( + BuiltinFunction( + function=partial(_attribute_access, class_access=class_access), + argnames=["self", "name"], + builtin=owner.__getattribute__, + ) + ) + + +def _getattr_function(context): + receiver = context.get_pyname("object") + if receiver is None: + return pyobjects.get_unknown() + value = receiver.get_object() + if value is None or value == pyobjects.get_unknown(): + return pyobjects.get_unknown() + owner = value if isinstance(value, pyobjects.AbstractClass) else value.get_type() + for name, native in ( + ("__getattr__", None), + ("__getattribute__", object.__getattribute__), + ): + hook = owner.get_attributes().get(name) + if ( + hook is not None + and getattr(hook.get_object(), "builtin", None) is not native + ): + return pyobjects.get_unknown() + if ( + isinstance(value, pyobjects.AbstractClass) + and value.get_metaclass() is not builtins["type"].get_object() + ): + return pyobjects.get_unknown() + name = context.get_argument("name") + default = context.get_pyname("default") + if ( + default is not None + and isinstance(name, LiteralString) + and name.value not in value + ): + # Without access hooks, a member absent from the receiver's model + # makes getattr() return its default. + return default.get_object() + return _attribute_access( + _CallContext( + ["self", "name"], + arguments.ObjectArguments([receiver, context.get_pyname("name")]), + ), + class_access=isinstance(value, pyobjects.AbstractClass), + ) + + class BuiltinObject(BuiltinClass): def __init__(self): - super().__init__(object, {}) + super().__init__( + object, + {"__getattribute__": _attribute_access_name(object, class_access=False)}, + ) + + +class NamespaceItems(pyobjects.PyObject): + """Source-proven namespace pairs; undeclared members remain unknown.""" + + def __init__(self, receiver, entries): + super().__init__(get_list_type(get_tuple(get_str(), pyobjects.get_unknown()))) + self.receiver = receiver + self.namespace = LiteralDict(get_str(), pyobjects.get_unknown(), entries) + + +def infer_runtime_call(function, args): + if not has_runtime_call_model(function): + return None + supplied = args.get_pynames(["object", "predicate"]) + receiver = supplied[0] if supplied else None + if receiver is None or len(supplied) > 1 and supplied[1] is not None: + return pyobjects.get_unknown() + value = receiver.get_object() + if value is None or value == pyobjects.get_unknown(): + return pyobjects.get_unknown() + owner = value if isinstance(value, pyobjects.AbstractClass) else value.get_type() + members = owner.get_attributes() + for name, native in ( + ("__getattr__", None), + ("__getattribute__", object.__getattribute__), + ("__dir__", object.__dir__), + ): + hook = members.get(name) + if ( + hook is not None + and getattr(hook.get_object(), "builtin", None) is not native + ): + return pyobjects.get_unknown() + if ( + isinstance(value, pyobjects.AbstractClass) + and value.get_metaclass() is not builtins["type"].get_object() + ): + return pyobjects.get_unknown() + entries = {} + for name in value.get_attributes(): + selected = _attribute_access( + _CallContext( + ["self", "name"], + arguments.ObjectArguments( + [ + receiver, + BuiltinName(LiteralString(name)), + ] + ), + ), + class_access=isinstance(value, pyobjects.AbstractClass), + ) + entries[name] = BuiltinName(selected) + return NamespaceItems(receiver, entries) + + +def has_runtime_call_model(function): + return _is_runtime_form(function, inspect.getmembers) class BuiltinType(BuiltinClass): def __init__(self): - super().__init__(type, {}) + constructor = BuiltinFunction( + function=self._new_type, + argnames=["metaclass", "name", "bases", "namespace"], + parent=self, + ) + super().__init__( + type, + { + "__new__": BuiltinName(constructor), + "__getattribute__": _attribute_access_name(type, class_access=True), + }, + ) + + @staticmethod + def is_constructor(function): + return ( + isinstance(function, BuiltinFunction) + and getattr(function.function, "__func__", None) is BuiltinType._new_type + ) + + def _new_type(self, context): + supplied = context.get_arguments(["metaclass", "name", "bases", "namespace"]) + if len(supplied) != 4: + return pyobjects.get_unknown() + metaclass, name, bases, namespace = supplied + if not isinstance(metaclass, pyobjects.AbstractClass): + return pyobjects.get_unknown() + if self not in metaclass.get_mro(): + raise TypeError("type.__new__ requires a subtype of type") + if not isinstance(name, LiteralString): + return pyobjects.get_unknown() + if bases is None or not isinstance(bases.get_type(), Tuple): + return pyobjects.get_unknown() + parents = bases.get_type().get_holding_objects() + if any(base is None or base == pyobjects.get_unknown() for base in parents): + return pyobjects.get_unknown() + if any(not isinstance(base, pyobjects.AbstractClass) for base in parents): + raise TypeError("type.__new__ bases must be classes") + if not isinstance(namespace, LiteralDict) or namespace.entries is None: + return pyobjects.get_unknown() + winner = metaclass + for base in parents: + candidate = base.get_metaclass() + if not isinstance(candidate, pyobjects.AbstractClass): + return pyobjects.get_unknown() + if winner in candidate.get_mro(): + winner = candidate + elif candidate not in winner.get_mro(): + raise TypeError("metaclass conflict") + if winner is not metaclass and not self.is_constructor( + winner["__new__"].get_object() + ): + return pyobjects.get_unknown() + result = pyobjects.ConstructedClass( + name.value, + parents or (builtins["object"].get_object(),), + winner, + namespace.entries, + ) + result.get_mro() + return result + + @staticmethod + def type_of(value): + """Resolve the one-argument operation without exposing kind markers.""" + if isinstance(value, pyobjects.AbstractClass): + return value.get_metaclass() + if isinstance(value, pyobjects.UnresolvedBase): + return value + if value is not None: + value_type = value.get_type() + if isinstance(value_type, BuiltinClass): + if inspect.isclass(value_type.builtin): + return _runtime_class(value_type.builtin) + elif isinstance(value_type, pyobjects.AbstractClass): + return value_type + return pyobjects.get_unknown() def _infer_sequence_for_pyname(pyname): @@ -788,12 +1541,55 @@ def _super_function(args): # pyclass = passed_self.get_type() pyclass = passed_class if isinstance(pyclass, pyobjects.AbstractClass): - supers = pyclass.get_superclasses() - if supers: - return pyobjects.PyObject(supers[0]) + mro = pyclass.get_consistent_mro() + if mro is None: + # Without a method resolution order, super() follows the + # declared bases, the first one taking precedence. + ancestors = pyclass.get_superclasses() + else: + ancestors = mro[1:] + if ancestors: + return pyobjects.PyObject(SuperType(pyclass, ancestors, passed_self)) return passed_self +class SuperType(pyobjects.AbstractClass): + """The type of ``super(cls, receiver)`` + + Attribute lookups follow the method resolution order of `cls` past + `cls` itself, as Python's ``super`` does. Methods reached through it + bind `receiver`, see `bound_receiver()`. + + """ + + def __init__(self, pyclass, ancestors, receiver): + super().__init__() + self.pyclass = pyclass + self.ancestors = ancestors + self.receiver = receiver + + def get_name(self): + return "super" + + def get_attributes(self): + result = {} + for ancestor in reversed(self.ancestors): + result.update(ancestor._get_own_attributes()) + return result + + +def bound_receiver(pyobject): + """Return the object a method called on `pyobject` binds + + A method reached through ``super(cls, receiver)`` binds `receiver`; + every other object binds itself. + + """ + if isinstance(pyobject.get_type(), SuperType): + return pyobject.get_type().receiver + return pyobject + + def _zip_function(args): args = args.get_pynames(["sequence"]) objects = [] @@ -826,11 +1622,45 @@ def _iter_function(args): return get_iterator(holding) +def _map_function(context): + supplied = context.get_pynames(["function", "iterable"]) + if len(supplied) < 2 or supplied[0] is None: + return pyobjects.get_unknown() + function, receiver = rope.base.evaluate.get_callable(supplied[0]) + if function is None: + return pyobjects.get_unknown() + items = [_infer_sequence_for_pyname(sequence) for sequence in supplied[1:]] + if any(item is None or item == pyobjects.get_unknown() for item in items): + return pyobjects.get_unknown() + passed = arguments.ObjectArguments([BuiltinName(item) for item in items]) + if receiver is not None: + passed = arguments.MixedArguments(receiver, passed, None) + result = infer_runtime_call(function, passed) + if result is None: + result = function.get_returned_object(passed) + return get_iterator(result) + + +def _next_function(context): + supplied = context.get_pynames(["iterator"]) + if not supplied or supplied[0] is None: + return pyobjects.get_unknown() + result = _infer_sequence_for_pyname(supplied[0]) + return result if result is not None else pyobjects.get_unknown() + + def _input_function(args): return get_str() _initial_builtins = { + "getattr": BuiltinName( + BuiltinFunction( + function=_getattr_function, + argnames=["object", "name", "default"], + builtin=getattr, + ) + ), "list": BuiltinName(get_list_type()), "dict": BuiltinName(get_dict_type()), "tuple": BuiltinName(get_tuple_type()), @@ -853,6 +1683,17 @@ def _input_function(args): "object": BuiltinName(BuiltinObject()), "type": BuiltinName(BuiltinType()), "iter": BuiltinName(BuiltinFunction(function=_iter_function, builtin=iter)), + "map": BuiltinName( + BuiltinClass( + map, + { + "__new__": BuiltinName( + BuiltinFunction(function=_map_function, builtin=map.__new__) + ) + }, + ) + ), + "next": BuiltinName(BuiltinFunction(function=_next_function, builtin=next)), "input": BuiltinName(BuiltinFunction(function=_input_function, builtin=input)), } diff --git a/rope/base/change.py b/rope/base/change.py index a6871f867..1e8eeca74 100644 --- a/rope/base/change.py +++ b/rope/base/change.py @@ -386,7 +386,7 @@ def _create_resource(self, file_name, kind="file"): def _get_destination_for_move(resource, destination): dest_path = resource.project._get_resource_path(destination) - if os.path.isdir(dest_path): + if resource.project.fscommands.isdir(dest_path): if destination != "": return destination + "/" + resource.name else: diff --git a/rope/base/classfunctions.py b/rope/base/classfunctions.py new file mode 100644 index 000000000..2fde895f9 --- /dev/null +++ b/rope/base/classfunctions.py @@ -0,0 +1,246 @@ +"""Bound access values for ordinary functions stored in a class body.""" + +from rope.base import arguments, ast, pynames, pyobjects + + +class BoundFunction(pyobjects.AbstractFunction): + def __init__(self, function, receiver): + super().__init__() + self.function = function + self.receiver = receiver + + def get_name(self): + return self.function.get_name() + + def get_doc(self): + return self.function.get_doc() + + def get_param_names(self, special_args=True): + return self.function.get_param_names(special_args) + + def bind_arguments(self, args): + return arguments.MixedArguments(self.receiver, args, None) + + def get_returned_object(self, args): + return self.function.get_returned_object(self.bind_arguments(args)) + + +class BoundAccessName(pynames.PyName): + def __init__(self, canonical, function, receiver): + self.canonical = canonical + self.value = BoundFunction(function, receiver) + + def get_object(self): + return self.value + + def get_definition_location(self): + return self.canonical.get_definition_location() + + def __eq__(self, other): + if isinstance(other, BoundAccessName): + other = other.canonical + return self.canonical == other + + def __hash__(self): + return hash(self.canonical) + + +def canonical_function(value): + if isinstance(value, BoundFunction): + return value.function + return value + + +def instance_access(primary, attribute, pyname, access_scope): + if not isinstance(pyname, pynames.AssignedName): + return pyname + instance = primary.get_object() + if isinstance(instance, pyobjects.AbstractClass): + return pyname + owner = instance.get_type() + if not isinstance(owner, pyobjects.PyClass): + return pyname + function = pyname.get_object() + if not isinstance(function, pyobjects.PyFunction): + return pyname + if not isinstance(function.parent, pyobjects.PyModule): + return pyname + node = function.get_ast() + if ( + not node.args.args + or node.args.posonlyargs + or node.args.kwonlyargs + or node.decorator_list + or getattr(node, "type_params", ()) + ): + return pyname + assignments = pyname.assignments + if len(assignments) != 1 or not isinstance(assignments[0].ast_node, ast.Name): + return pyname + assignment = assignments[0] + if assignment.levels or assignment.evaluation or assignment.assign_type: + return pyname + module, _ = pyname.get_definition_location() + if module is None: + return pyname + storage_scope = module.get_scope().get_inner_scope_for_line(assignment.get_lineno()) + if storage_scope.get_kind() != "Class": + return pyname + storage = storage_scope.pyobject + storage_node = storage.get_ast() + if storage_node.decorator_list or storage_node.keywords: + return pyname + if not any( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Name) + and statement.targets[0].id == attribute + and statement.value is assignment.ast_node + for statement in storage_node.body + ): + return pyname + modules = (module, owner.get_module(), access_scope.pyobject.get_module()) + for current in dict.fromkeys(modules): + if _has_override(current, primary, owner, storage, attribute): + return pyname + return BoundAccessName(pyname, function, primary) + + +def _has_override(module, primary, owner, storage, attribute): + # Instance dictionary functions remain ordinary unbound function values. + from rope.base import evaluate + + receiver = primary.get_object() + for target in module._get_source_binding_index().attribute_writes(attribute): + if not isinstance(target.value, ast.Name): + return True + scope = module.get_scope().get_inner_scope_for_line(target.lineno) + target_name = evaluate.eval_node(scope, target.value) + if target_name == primary: + return True + if target_name is not None: + target_object = target_name.get_object() + if ( + target_object is receiver + or target_object is owner + or target_object is storage + ): + return True + return False + + +def descriptor_access(primary, attribute, pyname, access_scope): + """Resolve a source descriptor that returns its unchanged constructor value.""" + if not isinstance(pyname, pynames.AssignedName) or len(pyname.assignments) != 1: + return pyname + receiver = primary.get_object() + owner = ( + receiver + if isinstance(receiver, pyobjects.AbstractClass) + else receiver.get_type() + ) + if not isinstance(owner, pyobjects.PyClass): + return pyname + assignment = pyname.assignments[0] + call = assignment.ast_node + if ( + not isinstance(call, ast.Call) + or assignment.levels + or assignment.evaluation + or assignment.assign_type + ): + return pyname + module, _ = pyname.get_definition_location() + if module is None: + return pyname + scope = module.get_scope().get_inner_scope_for_line(assignment.get_lineno()) + if scope.get_kind() != "Class" or scope.pyobject not in owner.get_mro(): + return pyname + from rope.base import evaluate + + constructor = evaluate.eval_node(scope, call.func) + if constructor is None or not isinstance( + constructor.get_object(), pyobjects.PyClass + ): + return pyname + descriptor = constructor.get_object() + declaration = descriptor.get_ast() + statements = declaration.body + if ( + statements + and isinstance(statements[0], ast.Expr) + and isinstance(statements[0].value, ast.Constant) + and isinstance(statements[0].value.value, str) + ): + statements = statements[1:] + # Any other class-body statement could rebind __init__ or __get__. + methods = { + node.name: node for node in statements if isinstance(node, ast.FunctionDef) + } + if ( + declaration.bases + or declaration.keywords + or declaration.decorator_list + or len(statements) != 2 + or set(methods) != {"__init__", "__get__"} + ): + return pyname + initializer, getter = methods["__init__"], methods["__get__"] + if initializer.decorator_list or getter.decorator_list: + return pyname + bodies = [] + for method in (initializer, getter): + body = method.body + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + body = body[1:] + bodies.append(body) + if len(bodies[0]) != 1 or len(bodies[1]) != 1: + return pyname + store, returned = bodies[0][0], bodies[1][0] + positional = (*initializer.args.posonlyargs, *initializer.args.args) + get_positional = (*getter.args.posonlyargs, *getter.args.args) + if len(positional) != 2 or len(get_positional) != 3: + return pyname + if ( + not isinstance(store, ast.Assign) + or len(store.targets) != 1 + or not isinstance(store.value, ast.Name) + ): + return pyname + target = store.targets[0] + if ( + not isinstance(target, ast.Attribute) + or not isinstance(target.value, ast.Name) + or target.value.id != positional[0].arg + or store.value.id != positional[1].arg + ): + return pyname + if ( + not isinstance(returned, ast.Return) + or not isinstance(returned.value, ast.Attribute) + or not isinstance(returned.value.value, ast.Name) + or returned.value.value.id != get_positional[0].arg + or returned.value.attr != target.attr + ): + return pyname + if any( + _has_override(current, primary, owner, scope.pyobject, attribute) + for current in dict.fromkeys( + (module, owner.get_module(), access_scope.pyobject.get_module()) + ) + ): + return pyname + function = descriptor["__init__"].get_object() + supplied = arguments.create_arguments( + None, function, call, scope, ignore_instance=True + ) + supplied = arguments.MixedArguments( + pynames.UnboundName(pyname.get_object()), supplied, scope + ) + value = supplied.get_pynames(function.get_param_names())[1] + return value if value is not None else pyname diff --git a/rope/base/codeanalyze.py b/rope/base/codeanalyze.py index d8b8da086..0f3c91944 100644 --- a/rope/base/codeanalyze.py +++ b/rope/base/codeanalyze.py @@ -1,7 +1,12 @@ +import ast import bisect +import io import re import token import tokenize +from typing import Iterator + +from rope.base import utils class ChangeCollector: @@ -72,7 +77,7 @@ def get_line_end(self, lineno): class ASTLinesAdapter: - """ + r""" Convert between stdlib's ast `col_offset` and rope's `region` offsets. ast `col_offset` are given in byte offset of the utf-8 source code from @@ -104,20 +109,37 @@ class LineNumberOutOfRange(IndexError): def __init__(self, source: str): self._code_str: str = source - self._code_bytes: bytes = source.encode("utf-8") + self._ascii: bool = source.isascii() + self._code_bytes: bytes | None self._starts_str: list[int] = self._initialize_line_starts( self.LINE_PATTERN_STR, self._code_str ) - self._starts_bytes: list[int] = self._initialize_line_starts( - self.LINE_PATTERN_BYTES, self._code_bytes - ) + if self._ascii: + # Byte offsets equal str offsets for ASCII sources, so both tables + # can share one list and the encoded source stays unneeded until a + # non-ASCII line is inspected (which cannot happen). + self._code_bytes = None + self._starts_bytes = self._starts_str + else: + self._code_bytes = source.encode("utf-8") + self._starts_bytes = self._initialize_line_starts( + self.LINE_PATTERN_BYTES, self._code_bytes + ) @classmethod def _initialize_line_starts(cls, pattern, source) -> list[int]: - matches = list(pattern.finditer(source)) - if len(matches) > 1 and matches[-1].start() == matches[-1].end(): - matches.pop() - starts = [m.start() for m in matches] + starts: list[int] = [] + append = starts.append + count = 0 + last_start = last_end = 0 + for match in pattern.finditer(source): + start = match.start() + append(start) + last_start = start + last_end = match.end() + count += 1 + if count > 1 and last_start == last_end: + starts.pop() if source[-1:] in [b"\n", b"\r", "\n", "\r"]: starts.append(len(source)) else: @@ -142,7 +164,10 @@ def _get_line_bytes(self, line_idx: int, col_offset: int | None = None) -> bytes end = self._starts_bytes[line_idx + 1] if col_offset is not None: end = min(end, start + col_offset) - return self._code_bytes[start:end].rstrip(b"\r\n") + code_bytes = self._code_bytes + if code_bytes is None: + code_bytes = self._code_bytes = self._code_str.encode("utf-8") + return code_bytes[start:end].rstrip(b"\r\n") def _is_line_ascii_only(self, line_idx: int): start_str = self._starts_str[line_idx] @@ -162,7 +187,11 @@ def _line_region_offset(self, line_idx: int, col_offset: int) -> int: def _absolute_region_offset(self, line_idx: int, col_offset: int) -> int: """str offset relative to the start of file""" - line_offset = self._line_region_offset(line_idx, col_offset) + if col_offset == 0 or self._ascii: + # ASCII sources (and column 0) need no per-line byte comparison. + line_offset = col_offset + else: + line_offset = self._line_region_offset(line_idx, col_offset) return self._starts_str[line_idx] + line_offset @classmethod @@ -208,9 +237,18 @@ def __getitem__(self, node) -> tuple[int, int] | tuple[None, None]: except AttributeError: return (None, None) - region_start = self._absolute_region_offset(line_idx, col_offset) - region_end = self._absolute_region_offset(end_line_idx, end_col_offset) - return (region_start, region_end) + starts_str = self._starts_str + if self._ascii: + # Every line is ASCII-only, so `_line_region_offset` is the + # column itself for both boundaries. + return ( + starts_str[line_idx] + col_offset, + starts_str[end_line_idx] + end_col_offset, + ) + return ( + self._absolute_region_offset(line_idx, col_offset), + self._absolute_region_offset(end_line_idx, end_col_offset), + ) class ArrayLinesAdapter: @@ -318,7 +356,7 @@ def logical_line_in(self, line_number): except IndentationError as e: tries += 1 if tries == 5: - raise e + raise lineno = e.lineno + block_start - 1 indents = count_line_indents(self.lines.get_line(lineno)) @@ -533,3 +571,54 @@ def get_any_string_pattern(): def get_comment_pattern(): return r"#[^\n]*" + + +def name_offsets(source: str, name: str) -> tuple[int, ...]: + """Return native Python NAME token offsets without exposing the cache.""" + return _name_offsets(source).get(name, ()) + + +@utils.cached(16) +def _name_offsets(source: str) -> dict: + """Map every NAME token of `source` to the offsets where it occurs + + The source is tokenized once however many names are searched in it. + Complete string tokens expose replacement expressions through Python's + native AST; expression tokens retain their source coordinates. The + result is keyed by the source text, so a changed source is indexed again. + """ + result = {} + for name, offset in _token_names(source): + result.setdefault(name, []).append(offset) + return {name: tuple(offsets) for name, offsets in result.items()} + + +def _token_names(source: str) -> Iterator: + line_starts = [0] + for line in source.splitlines(keepends=True): + line_starts.append(line_starts[-1] + len(line)) + for item in tokenize.generate_tokens(io.StringIO(source).readline): + row, column = item.start + if item.type == tokenize.NAME: + yield item.string, line_starts[row - 1] + column + elif item.type == tokenize.STRING and item.string.lower().startswith( + ("f", "rf", "fr") + ): + module = ast.parse(item.string) + positions = ASTLinesAdapter(item.string) + for name, offset in _formatted_names( + item.string, module.body[0].value, positions + ): + yield name, line_starts[row - 1] + column + offset + + +def _formatted_names(source, node, positions) -> Iterator: + for value in node.values: + if not isinstance(value, ast.FormattedValue): + continue + start, end = positions[value.value] + # AST spans omit the grouping that permits continuation indentation. + for name, offset in _token_names("(" + source[start:end] + ")"): + yield name, start + offset - 1 + if value.format_spec is not None: + yield from _formatted_names(source, value.format_spec, positions) diff --git a/rope/base/evaluate.py b/rope/base/evaluate.py index 4c7580cce..6aa446464 100644 --- a/rope/base/evaluate.py +++ b/rope/base/evaluate.py @@ -1,3 +1,6 @@ +import operator +import sys +import typing from operator import itemgetter from typing import Optional, Tuple @@ -7,16 +10,193 @@ from rope.base import ( arguments, ast, + classfunctions, + codeanalyze, exceptions, nameanalyze, pyobjects, pyobjectsdef, + pyscopes, + utils, worder, ) BadIdentifierError = exceptions.BadIdentifierError +def _has_stdlib_import_prelude(module, node): + """Certify a direct header without loading its imported modules.""" + project = module.pycore.project + resource = module.get_resource() + folder = resource.parent if resource is not None else None + for statement in module.get_ast().body: + if statement is node: + return True + if ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + and isinstance(statement.value.value, str) + ): + continue + if isinstance(statement, ast.Import): + imported = [alias.name for alias in statement.names] + elif isinstance(statement, ast.ImportFrom) and not statement.level: + if any(alias.name == "*" for alias in statement.names): + return False + imported = [statement.module] + else: + return False + for name in imported: + if not name or name.partition(".")[0] not in sys.stdlib_module_names: + return False + runtime = sys.modules.get(name) + if type(runtime) is not type(sys): + return False + if name in sys.builtin_module_names: + continue + runtime_file = vars(runtime).get("__file__") + origin = project.find_module(name, folder) + if origin is None: + return False + if origin.is_folder(): + if not origin.has_child("__init__.py"): + return False + origin = origin.get_child("__init__.py") + if not utils.same_file_path(origin.real_path, runtime_file): + return False + return False + + +def _header_version_guard(module, names, node): + """Prove a header sys.version_info comparison; return None when unproved. + + The current native bindings and an authenticated standard-library prelude + supply the runtime. No target package or expression is executed. Other + guards, scopes and comparison domains keep their existing treatment. + """ + if module is None: + return None + if not all( + isinstance(statement, (ast.Import, ast.ImportFrom)) + for statement in node.body + node.orelse + ): + return None + test = node.test + if not ( + isinstance(test, ast.Compare) + and len(test.ops) == len(test.comparators) == 1 + and isinstance(test.left, ast.Attribute) + and test.left.attr == "version_info" + and isinstance(test.left.value, ast.Name) + and isinstance(test.comparators[0], ast.Tuple) + ): + return None + binding = names.get(test.left.value.id) + if not ( + isinstance(binding, rope.base.pynames.ImportedModule) + and binding.importing_module is module + and binding.module_name == "sys" + and binding.level == 0 + and _has_stdlib_import_prelude(module, node) + ): + return None + namespace = binding.get_object() + if not ( + isinstance(namespace, rope.base.builtins.BuiltinModule) + and namespace.name == "sys" + and namespace.module is sys + ): + return None + version = namespace.get_attribute("version_info").get_object() + if not ( + isinstance(version, rope.base.builtins.BuiltinUnknown) + and version.builtin is sys.version_info + ): + return None + try: + compared = ast.literal_eval(test.comparators[0]) + except (TypeError, ValueError): + return None + if any( + type(value) is not type(actual) + for value, actual in zip(compared, version.builtin) + ): + return None + comparison = { + ast.Lt: operator.lt, + ast.LtE: operator.le, + ast.Gt: operator.gt, + ast.GtE: operator.ge, + ast.Eq: operator.eq, + ast.NotEq: operator.ne, + }.get(type(test.ops[0])) + if comparison is None: + return None + return comparison(version.builtin, compared) + + +class SourceValueScope(pyscopes.TemporaryScope): + """A call's value overlay, without replacing canonical source binders.""" + + def __init__( + self, + canonical_scope, + capture_scope, + names, + *, + branch_context=(), + source_epoch=None, + ): + super().__init__(canonical_scope.pycore, canonical_scope, names) + self.pyobject = canonical_scope.pyobject + self.canonical_scope = canonical_scope + self.capture_scope = capture_scope + self.branch_context = branch_context + self.source_epoch = source_epoch + + def lookup(self, name, *, read_node=None): + if name in self.names: + return self.names[name] + if self.source_epoch is not None: + from rope.base import pynamesdef + from rope.base.sourcebindings import SourceValueName + + binding = self.canonical_scope.lookup(name) + if isinstance(binding, pynamesdef.AssignedName): + return SourceValueName( + binding, binding.get_object(read_node=self.source_epoch) + ) + return binding + if name in self.canonical_scope.get_names(): + binding = self.canonical_scope.get_names()[name] + from rope.base import pynamesdef + from rope.base.sourcebindings import SourceLocalName, SourceValueName + + if ( + isinstance(binding, pynamesdef.ParameterName) + and binding.assigned_value is not None + and read_node is not None + ): + return SourceValueName( + binding, + binding.assigned_value.get_object( + branch_context=self.branch_context, read_node=read_node + ), + ) + if self.branch_context and isinstance( + binding, (pynamesdef.AssignedName, SourceLocalName) + ): + return SourceValueName( + binding, + binding.get_object(branch_context=self.branch_context), + ) + return binding + return self.capture_scope.lookup(name) + + def _propagated_lookup(self, name): + return self.lookup(name) + + def eval_location(pymodule, offset): """Find the pyname at the offset""" return eval_location2(pymodule, offset)[1] @@ -37,11 +217,64 @@ def eval_node(scope, node): def eval_node2(scope, node): + module = scope.pyobject.get_module() + context = module._get_source_binding_index().context_for_node(node) + if context is not None and scope.get_kind() != "Temporary": + scope = context.environment + parameter_facts = False + owner = context.structural_scope.pyobject if context is not None else scope.pyobject + if isinstance(owner, pyobjectsdef.PyFunction): + owner.get_scope().get_names() + parameter_facts = any( + parameter.assigned_value is not None + for parameter in owner.get_parameters().values() + ) + if context is not None and (context.branch_context or parameter_facts): + if isinstance(scope, SourceValueScope): + if scope.pyobject is context.environment.pyobject: + branches = dict(scope.branch_context) + branches.update(context.branch_context) + scope = SourceValueScope( + scope.canonical_scope, + scope.capture_scope, + scope.names, + branch_context=tuple(branches.items()), + source_epoch=scope.source_epoch, + ) + elif scope.get_kind() != "Temporary": + scope = SourceValueScope( + scope, scope, {}, branch_context=context.branch_context + ) evaluator = StatementEvaluator(scope) evaluator.visit(node) + if ( + evaluator.result is not None + and evaluator.old_result is not None + and isinstance(evaluator.old_result.get_object(), pyobjects.SampledValue) + ): + from rope.base.sourcebindings import SourceValueName + + evaluator.result = SourceValueName( + evaluator.result, + pyobjects.sampled_value( + evaluator.result.get_object(), evaluator.old_result.get_object() + ), + ) return evaluator.old_result, evaluator.result +def get_callable(pyname): + """Return the function and receiver selected by a callable value.""" + value = pyname.get_object() + if isinstance(value, pyobjects.AbstractFunction): + return value, None + if "__call__" in value: + function = value["__call__"].get_object() + if isinstance(function, pyobjects.AbstractFunction): + return function, pyname + return None, None + + def eval_str(holding_scope, name): return eval_str2(holding_scope, name)[1] @@ -90,33 +323,93 @@ def get_pyname_at(self, offset): def get_primary_and_pyname_at( self, offset: int, + ) -> Tuple[Optional[rope.base.pynames.PyName], Optional[rope.base.pynames.PyName]]: + # Occurrence searches resolve the same (module, offset) many times + # while the project data does not change. The memo lives on the + # PyModule and is stamped with the project-wide data epoch, so any + # resource invalidation (forget_all_data) retires every entry. + pymodule = self.module_scope.pyobject + cache = getattr(getattr(pymodule, "pycore", None), "module_cache", None) + epoch = getattr(cache, "data_epoch", None) + memo = getattr(pymodule, "_offset_resolution_memo", None) + if epoch is not None and memo is not None and memo[0] == epoch: + try: + return memo[1][offset] + except KeyError: + pass + result = self._resolve_primary_and_pyname_at(offset) + if epoch is not None: + if memo is None or memo[0] != epoch: + memo = (epoch, {}) + pymodule._offset_resolution_memo = memo + memo[1][offset] = result + return result + + def _resolve_primary_and_pyname_at( + self, + offset: int, ) -> Tuple[Optional[rope.base.pynames.PyName], Optional[rope.base.pynames.PyName]]: lineno = self.lines.get_line_number(offset) - holding_scope = self.module_scope.get_inner_scope_for_offset(offset) + index = self.module_scope.pyobject._get_source_binding_index() + context = index.context_at(offset) + holding_scope = context.environment + structural_scope = context.structural_scope + parameter = index.binding_for_token(offset) + if parameter is not None: + return (None, parameter) # function keyword parameter - if self.worder.is_function_keyword_parameter(offset): + if self.worder.is_function_keyword_parameter(offset) and not ( + isinstance(context.source_node, ast.Name) + and isinstance(context.source_node.ctx, ast.Store) + ): keyword_name = self.worder.get_word_at(offset) pyobject = self.get_enclosing_function(offset) - if isinstance(pyobject, pyobjectsdef.PyFunction): - parameter_name = pyobject.get_parameters().get(keyword_name, None) + from rope.base.sourcebindings import SourceLambdaFunction + + if isinstance(pyobject, SourceLambdaFunction): + parameter_name = None + if keyword_name in pyobject.get_keyword_param_names(): + parameter_name = pyobject.get_parameters().get(keyword_name) + return (None, parameter_name) + elif isinstance(pyobject, pyobjectsdef.PyFunction): + parameter_name = None + if keyword_name in pyobject.get_keyword_param_names(): + parameter_name = pyobject.get_parameters().get(keyword_name) return (None, parameter_name) elif isinstance(pyobject, pyobjects.AbstractFunction): parameter_name = rope.base.pynames.ParameterName() return (None, parameter_name) + return (None, None) + # A class data overlay must not replace the imported source symbol. + if holding_scope.get_kind() == "Class": + from rope.base import externaldata + + imported = externaldata.import_source_name( + holding_scope, + lineno, + offset, + self.worder.get_word_at(offset), + self.worder.get_primary_at(offset), + self.worder.get_from_module(offset), + self.worder.is_from_aliased(offset), + self.worder.is_from_statement_module(offset), + ) + if imported is not None: + return (None, imported) # class body - if self._is_defined_in_class_body(holding_scope, offset, lineno): - class_scope = holding_scope - if lineno == holding_scope.get_start(): - class_scope = holding_scope.parent + if self._is_defined_in_class_body(structural_scope, offset, lineno): + class_scope = structural_scope + if lineno == structural_scope.get_start(): + class_scope = structural_scope.parent name = self.worder.get_primary_at(offset).strip() try: return (None, class_scope.pyobject[name]) except rope.base.exceptions.AttributeNotFoundError: return (None, None) # function header - if self._is_function_name_in_function_header(holding_scope, offset, lineno): + if self._is_function_name_in_function_header(structural_scope, offset, lineno): name = self.worder.get_primary_at(offset).strip() - return (None, holding_scope.parent[name]) + return (None, structural_scope.parent[name]) # module in a from statement or an imported name that is aliased if self.worder.is_from_statement_module( offset @@ -128,16 +421,56 @@ def get_primary_and_pyname_at( name = self.worder.get_from_aliased(offset) else: name = self.worder.get_primary_at(offset) - return eval_str2(holding_scope, name) + primary, pyname = eval_str2(holding_scope, name) + from rope.base.sourcebindings import SourceValueName + + if isinstance(pyname, SourceValueName): + pyname = pyname.source_pyname + return primary, pyname + + def _get_parameter_at(self, scope, offset): + if scope.get_kind() != "Function": + return None + function = scope.pyobject + args = function.arguments + nodes = ( + args.posonlyargs + args.args + args.kwonlyargs + [args.vararg, args.kwarg] + ) + for node in nodes: + if node is None: + continue + prefix = self.lines.get_line(node.lineno).encode("utf-8")[: node.col_offset] + start = self.lines.get_line_start(node.lineno) + len(prefix.decode("utf-8")) + _, end = self.worder.get_word_range(start) + if start <= offset < end: + return function.get_parameters()[node.arg] + return None def get_enclosing_function(self, offset): + module = self.module_scope.pyobject + lines = codeanalyze.ASTLinesAdapter(module.source_code) + for node in ast.walk(module.get_ast()): + if not isinstance(node, ast.Call): + continue + for keyword in node.keywords: + if keyword.arg is None: + continue + start, _ = lines[keyword] + if start <= offset < start + len(keyword.arg): + return self._callable_object( + eval_node(self.module_scope, node.func) + ) function_parens = self.worder.find_parens_start_from_inside(offset) try: function_pyname = self.get_pyname_at(function_parens - 1) except BadIdentifierError: function_pyname = None + return self._callable_object(function_pyname) + + @staticmethod + def _callable_object(function_pyname): if function_pyname is not None: - pyobject = function_pyname.get_object() + pyobject = classfunctions.canonical_function(function_pyname.get_object()) if isinstance(pyobject, pyobjects.AbstractFunction): return pyobject elif ( @@ -164,7 +497,34 @@ def __init__(self, scope): self.old_result = None def _Name(self, node): - self.result = self.scope.lookup(node.id) + if isinstance(self.scope, SourceValueScope): + self.result = self.scope.lookup(node.id, read_node=node) + else: + self.result = self.scope.lookup(node.id) + + def _IfExp(self, node): + if isinstance(node.test, ast.Constant): + self.old_result, self.result = eval_node2( + self.scope, node.body if node.test.value else node.orelse + ) + return + left_primary, left = eval_node2(self.scope, node.body) + right_primary, right = eval_node2(self.scope, node.orelse) + same_receiver = left_primary is right_primary or ( + left_primary is not None + and right_primary is not None + and left_primary.get_object() is right_primary.get_object() + ) + if ( + left is not None + and right is not None + and left.get_object() is right.get_object() + and same_receiver + ): + self.result = left + self.old_result = left_primary + else: + self.result = rope.base.pynames.UnboundName() def _Attribute(self, node): pyname = eval_node(self.scope, node.value) @@ -174,48 +534,162 @@ def _Attribute(self, node): if pyname.get_object() != rope.base.pyobjects.get_unknown(): try: self.result = pyname.get_object()[node.attr] + self.result = classfunctions.instance_access( + pyname, node.attr, self.result, self.scope + ) + self.result = classfunctions.descriptor_access( + pyname, node.attr, self.result, self.scope + ) + from rope.base.sourcebindings import ( + SourceLambdaFunction, + SourceValueName, + source_lambda_attribute_value, + ) + + selected = source_lambda_attribute_value(pyname, self.result) + if selected is not None: + self.result = selected + function = self.result.get_object() + if isinstance(function, SourceLambdaFunction): + receiver = function.descriptor_receiver(pyname, self.result) + if receiver is not None: + self.result = SourceValueName( + self.result, function.bind_receiver(receiver) + ) except exceptions.AttributeNotFoundError: self.result = None def _Call(self, node): - primary, pyobject = self._get_primary_and_object_for_node(node.func) + primary, func_pyname = eval_node2(self.scope, node.func) + pyobject = None + if func_pyname is not None: + pyobject = func_pyname.get_object() if pyobject is None: return + self.old_result = ( + func_pyname if isinstance(pyobject, pyobjects.SampledValue) else primary + ) + if not isinstance(pyobject, pyobjects.SampledValue): + for argument in (*node.args, *node.keywords): + expression = ( + argument.value + if isinstance(argument, (ast.Starred, ast.keyword)) + else argument + ) + binding = eval_node(self.scope, expression) + if binding is not None and isinstance( + binding.get_object(), pyobjects.SampledValue + ): + self.old_result = binding + break + + if ( + isinstance(pyobject, rope.base.builtins.BuiltinType) + and len(node.args) == 1 + and not node.keywords + and not isinstance(node.args[0], ast.Starred) + ): + binding = eval_node(self.scope, node.args[0]) + self.result = rope.base.pynames.UnboundName( + pyobject.type_of(binding.get_object() if binding is not None else None) + ) + return def _get_returned(pyobject): - args = arguments.create_arguments(primary, pyobject, node, self.scope) + args = arguments.create_arguments( + primary, + pyobject, + node, + self.scope, + ) + modeled = rope.base.builtins.infer_runtime_call(pyobject, args) + if modeled is not None: + return modeled return pyobject.get_returned_object(args) + if isinstance(pyobject, pyobjects.SampledValue) and isinstance( + pyobject.representative, pyobjects.AbstractFunction + ): + self.result = rope.base.pynames.UnboundName( + _get_returned(pyobject.representative) + ) + return + if isinstance(pyobject, rope.base.pyobjects.AbstractClass): + from rope.base.sourcebindings import SourceLambdaFunction + + if ( + isinstance(pyobject, rope.base.builtins.BuiltinClass) + and pyobject.builtin in (staticmethod, classmethod) + and node.args + ): + function = eval_node(self.scope, node.args[0]) + if function is not None and isinstance( + function.get_object(), SourceLambdaFunction + ): + self.result = rope.base.pynames.UnboundName( + function.get_object().as_descriptor(pyobject.builtin.__name__) + ) + return result = None if "__new__" in pyobject: new_function = pyobject["__new__"].get_object() + if rope.base.builtins.BuiltinType.is_constructor(new_function) or ( + isinstance(new_function, rope.base.pyobjects.PyFunction) + and new_function.get_kind() != "classmethod" + ): + args = arguments.create_arguments( + None, new_function, node, self.scope, ignore_instance=True + ) + args = arguments.MixedArguments( + rope.base.pynames.UnboundName(pyobject), args, self.scope + ) + self.result = rope.base.pynames.UnboundName( + new_function.get_returned_object(args) + ) + return result = _get_returned(new_function) if result is None or result == rope.base.pyobjects.get_unknown(): result = rope.base.pyobjects.PyObject(pyobject) self.result = rope.base.pynames.UnboundName(pyobject=result) return - pyfunction = None - if isinstance(pyobject, rope.base.pyobjects.AbstractFunction): - pyfunction = pyobject - elif "__call__" in pyobject: - pyfunction = pyobject["__call__"].get_object() + pyfunction, receiver = get_callable(func_pyname) if pyfunction is not None: - self.result = rope.base.pynames.UnboundName( - pyobject=_get_returned(pyfunction) - ) + if receiver is None: + result = _get_returned(pyfunction) + else: + args = arguments.create_arguments( + None, pyfunction, node, self.scope, ignore_instance=True + ) + args = arguments.bind_method( + receiver, + pyfunction, + args, + self.scope, + owner=receiver.get_object().get_type(), + ) + result = pyfunction.get_returned_object(args) + self.result = rope.base.pynames.UnboundName(pyobject=result) def _Str(self, node): self.result = rope.base.pynames.UnboundName( - pyobject=rope.base.builtins.get_str() + pyobject=rope.base.builtins.LiteralString(node.s) ) def _Num(self, node): type_name = type(node.n).__name__ self.result = self._get_builtin_name(type_name) + def _JoinedStr(self, node): + self.result = self._get_builtin_name("str") + def _Constant(self, node): + if isinstance(node.value, str): + self.result = rope.base.pynames.UnboundName( + rope.base.builtins.LiteralString(node.value) + ) + return type_name = type(node.value).__name__ try: self.result = self._get_builtin_name(type_name) @@ -228,9 +702,30 @@ def _get_builtin_name(self, type_name): return rope.base.pynames.UnboundName(rope.base.pyobjects.PyObject(pytype)) def _BinOp(self, node): - self.result = rope.base.pynames.UnboundName( - self._get_object_for_node(node.left) - ) + left = self._get_object_for_node(node.left) + if isinstance(node.op, ast.BitOr): + right = self._get_object_for_node(node.right) + parts = (node.left, node.right) + values = (left, right) + if any( + isinstance( + value, (pyobjects.AbstractClass, rope.base.builtins.GenericAlias) + ) + for value in values + ): + from rope.base.oi.type_hinting.utils import _hint_value, union_type + + alternatives = tuple( + ( + rope.base.builtins._runtime_class(type(None)) + if isinstance(part, ast.Constant) and part.value is None + else _hint_value(value) + ) + for part, value in zip(parts, values) + ) + self.result = rope.base.pynames.UnboundName(union_type(alternatives)) + return + self.result = rope.base.pynames.UnboundName(left) def _BoolOp(self, node): pyobject = self._get_object_for_node(node.values[0]) @@ -260,8 +755,24 @@ def _Dict(self, node): keys = self._get_object_for_node(keys) if values: values = self._get_object_for_node(values) + entries = {} + for key, value in zip(node.keys, node.values): + if key is None: + entries = None + break + key_object = self._get_object_for_node(key) + if not isinstance(key_object, rope.base.builtins.LiteralString): + entries = None + break + binding = eval_node(self.scope, value) + if binding is None: + entries = None + break + entries[key_object.value] = rope.base.pynames.UnboundName( + binding.get_object() + ) self.result = rope.base.pynames.UnboundName( - pyobject=rope.base.builtins.get_dict(keys, values) + pyobject=rope.base.builtins.LiteralDict(keys, values, entries) ) def _List(self, node): @@ -284,12 +795,76 @@ def _GeneratorExp(self, node): pyobject=rope.base.builtins.get_iterator(pyobject) ) + def _SetComp(self, node): + pyobject = self._what_does_comprehension_hold(node) + self.result = rope.base.pynames.UnboundName( + pyobject=rope.base.builtins.get_set(pyobject) + ) + + def _DictComp(self, node): + scope = self._make_comprehension_scope(node) + key = eval_node(scope, node.key) + value = eval_node(scope, node.value) + self.result = rope.base.pynames.UnboundName( + pyobject=rope.base.builtins.get_dict( + key.get_object() if key is not None else None, + value.get_object() if value is not None else None, + ) + ) + def _what_does_comprehension_hold(self, node): scope = self._make_comprehension_scope(node) pyname = eval_node(scope, node.elt) return pyname.get_object() if pyname is not None else None def _make_comprehension_scope(self, node): + module = self.scope.pyobject.get_module() + binding_scope = module._get_source_binding_index().comprehension_scope(node) + if binding_scope is not None: + if self.scope.get_kind() == "Temporary": + values = { + name: rope.base.pynames.UnboundName() + for name in binding_scope.get_names() + } + value_scope = SourceValueScope(binding_scope, self.scope, values) + for number, generator in enumerate(node.generators): + entry = self.scope if number == 0 else value_scope + iterator = eval_node(entry, generator.iter) + if iterator is None: + continue + iterator_scope = pyscopes.TemporaryScope( + entry.pycore, entry, {"__rope_iterable": iterator} + ) + element = eval_str( + iterator_scope, "__rope_iterable.__iter__().next()" + ) + if element is None: + continue + for name, levels in nameanalyze.get_name_levels(generator.target): + value = element + for level in levels: + element_type = value.get_object().get_type() + if isinstance(element_type, rope.base.builtins.Tuple): + holdings = element_type.get_holding_objects() + value = ( + rope.base.pynames.UnboundName(holdings[level]) + if level < len(holdings) + and holdings[level] is not None + else None + ) + else: + item_scope = pyscopes.TemporaryScope( + entry.pycore, entry, {"__rope_element": value} + ) + value = eval_str( + item_scope, "__rope_element[%s]" % level + ) + if value is None: + break + if value is not None: + values[name] = value + return value_scope + return binding_scope scope = self.scope module = scope.pyobject.get_module() names = {} @@ -301,13 +876,7 @@ def _make_comprehension_scope(self, node): return rope.base.pyscopes.TemporaryScope(scope.pycore, scope, names) def _Tuple(self, node): - objects = [] - if len(node.elts) < 4: - for stmt in node.elts: - pyobject = self._get_object_for_node(stmt) - objects.append(pyobject) - else: - objects.append(self._get_object_for_node(node.elts[0])) + objects = [self._get_object_for_node(stmt) for stmt in node.elts] self.result = rope.base.pynames.UnboundName( pyobject=rope.base.builtins.get_tuple(*objects) ) @@ -319,20 +888,104 @@ def _get_object_for_node(self, stmt): pyobject = pyname.get_object() return pyobject - def _get_primary_and_object_for_node(self, stmt): - primary, pyname = eval_node2(self.scope, stmt) - pyobject = None - if pyname is not None: - pyobject = pyname.get_object() - return primary, pyobject - def _Subscript(self, node): - if isinstance(node.slice, ast.Index): - self._call_function(node.value, "__getitem__", [node.slice.value]) - elif isinstance(node.slice, ast.Slice): - self._call_function(node.value, "__getitem__", [node.slice]) - elif isinstance(node.slice, ast.expr): - self._call_function(node.value, "__getitem__", [node.value]) + receiver = eval_node(self.scope, node.value) + if receiver is None: + return + self.old_result = receiver + value = receiver.get_object() + is_class = isinstance(value, pyobjects.AbstractClass) + owner = value.get_metaclass() if is_class else value.get_type() + if is_class and not isinstance(owner, pyobjects.AbstractClass): + self.result = rope.base.pynames.UnboundName(owner) + return + if rope.base.builtins._runtime_unpack(value) is not None: + # PEP 646: an explicit ``Unpack[Ts]`` evaluates to the typing + # alias carrying the parameter, mirroring the runtime's + # ``_GenericAlias``; the generic-alias validation recognizes + # the form through its origin. + key = node.slice.value if isinstance(node.slice, ast.Index) else node.slice + if isinstance(key, ast.Starred): + # The runtime rejects starred subscripting of ``Unpack``; + # rope models that failure as no result. + return + target = eval_node(self.scope, key) + if target is None: + return + module = self.scope.pyobject.get_module() + self.result = rope.base.pynames.UnboundName( + rope.base.builtins.GenericAlias( + value, (target.get_object(),), module.pycore.project, True + ) + ) + return + if isinstance(owner, rope.base.builtins.Tuple): + try: + if isinstance(node.slice, ast.Slice): + bounds = ( + ast.literal_eval(bound) if bound is not None else None + for bound in ( + node.slice.lower, + node.slice.upper, + node.slice.step, + ) + ) + result = rope.base.builtins.get_tuple( + *owner.get_holding_objects()[slice(*bounds)] + ) + else: + key = ast.literal_eval(node.slice) + result = owner.get_item(key) if isinstance(key, int) else None + except (ValueError, TypeError): + result = None + self.result = rope.base.pynames.UnboundName(result) + return + if "__getitem__" in owner: + called, callable_receiver = get_callable(owner["__getitem__"]) + descriptor_owner = owner + elif is_class and "__class_getitem__" in value: + called, callable_receiver = get_callable(value["__class_getitem__"]) + descriptor_owner = value + else: + return + if called is None: + return + if callable_receiver is not None: + receiver = callable_receiver + descriptor_owner = receiver.get_object().get_type() + key = node.slice.value if isinstance(node.slice, ast.Index) else node.slice + starred = isinstance(key, ast.Starred) + if starred: + # PEP 646: the starred subscription syntax exists only at the + # expression level; evaluate the target so the typing alias + # validation receives the unpack fact, not the star node. + key = key.value + args = arguments.Arguments([key], self.scope, called) + args = arguments.bind_method( + receiver, called, args, self.scope, owner=descriptor_owner + ) + if starred: + target = eval_node(self.scope, key) + if target is None or not rope.base.builtins._is_runtime_class( + target.get_object().get_type(), typing.TypeVarTuple + ): + # Runtime rejects every non-TypeVarTuple starred subscript; + # rope models that failure as no result. + return + args = arguments.bind_method( + receiver, + called, + arguments.ObjectArguments( + [ + rope.base.pynames.UnboundName( + rope.base.builtins.UnpackedTypeVarTuple(target.get_object()) + ) + ] + ), + self.scope, + owner=descriptor_owner, + ) + self.result = rope.base.pynames.UnboundName(called.get_returned_object(args)) def _Slice(self, node): self.result = self._get_builtin_name("slice") @@ -350,15 +1003,19 @@ def _call_function(self, node, function_name, other_args=None): args = [node] if other_args: args += other_args - arguments_ = arguments.Arguments(args, self.scope) + arguments_ = arguments.Arguments(args, self.scope, called) self.result = rope.base.pynames.UnboundName( pyobject=called.get_returned_object(arguments_) ) def _Lambda(self, node): - self.result = rope.base.pynames.UnboundName( - pyobject=rope.base.builtins.Lambda(node, self.scope) - ) + module = self.scope.pyobject.get_module() + function = module._get_source_binding_index().callable_for_node(node) + if function is None: + function = rope.base.builtins.Lambda(node, self.scope) + elif self.scope.get_kind() == "Temporary": + function = function.bind_creation_scope(self.scope) + self.result = rope.base.pynames.UnboundName(pyobject=function) def _get_evaluated_names(targets, assigned, module, evaluation, lineno): diff --git a/rope/base/exceptions.py b/rope/base/exceptions.py index af2d15838..bc06794b7 100644 --- a/rope/base/exceptions.py +++ b/rope/base/exceptions.py @@ -34,6 +34,15 @@ class BadIdentifierError(RopeError): """The name cannot be resolved""" +class InconsistentHierarchyError(RopeError): + """A class hierarchy has no method resolution order + + Python refuses to create such a class: its bases are cyclic, + repeated, or cannot be merged by the C3 linearization. + + """ + + class ModuleSyntaxError(RopeError): """Module has syntax errors diff --git a/rope/base/externaldata.py b/rope/base/externaldata.py new file mode 100644 index 000000000..55d62c4b7 --- /dev/null +++ b/rope/base/externaldata.py @@ -0,0 +1,1002 @@ +"""Conservative same-module class data writes, without descriptor inference.""" + +import collections +import io +import tokenize +from collections import Counter, defaultdict, deque + +from rope.base import ( + ast, + codeanalyze, + exceptions, + pynames, + pynamesdef, + pyobjects, + utils, +) + +_CLASS_HOOKS = frozenset( + { + "__get__", + "__set__", + "__delete__", + "__new__", + "__getattribute__", + "__getattr__", + "__setattr__", + "__delattr__", + } +) +_VALUE_HOOKS = frozenset({"__getattribute__", "__getattr__", "__new__", "__call__"}) + + +def _scope_nodes(body): + pending = list(body) + while pending: + node = pending.pop() + yield node + if isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda) + ): + continue + if isinstance(node, ast.comprehension): + pending.extend([node.iter, *node.ifs]) + else: + pending.extend(ast.iter_child_nodes(node)) + + +def _class_bindings(node): + names, unmergeable = set(), set() + for child in _scope_nodes(node.body): + if isinstance(child, ast.Name) and isinstance(child.ctx, (ast.Store, ast.Del)): + names.add(child.id) + elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(child.name) + unmergeable.add(child.name) + elif isinstance(child, (ast.Import, ast.ImportFrom)): + bound = { + item.asname + or ( + item.name.split(".")[0] + if isinstance(child, ast.Import) + else item.name + ) + for item in child.names + } + names.update(bound) + elif isinstance(child, ast.ExceptHandler) and child.name: + names.add(child.name) + elif isinstance(child, (ast.MatchAs, ast.MatchStar)) and child.name: + names.add(child.name) + elif isinstance(child, ast.MatchMapping) and child.rest: + names.add(child.rest) + return names, unmergeable + + +def _walk_with_parents(tree): + """Return the module's nodes in ast.walk order together with a parent map.""" + nodes = [] + parents = {} + pending = collections.deque([tree]) + while pending: + node = pending.popleft() + nodes.append(node) + for child in ast.iter_child_nodes(node): + parents[child] = node + pending.append(child) + return nodes, parents + + +def _count_module_bindings(tree): + """Count module-level name bindings exactly like _Bindings visiting tree. + + Definitions and lambdas are not descended into, mirroring the visitor's + missing generic_visit calls; everything else is counted and traversed. + """ + counts = Counter() + wildcard_import = False + stack = [tree] + while stack: + node = stack.pop() + if isinstance(node, ast.Lambda): + continue + if isinstance(node, ast.Name): + if isinstance(node.ctx, (ast.Store, ast.Del)): + counts[node.id] += 1 + elif isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + counts[node.name] += 1 + continue + elif isinstance(node, ast.Import): + for item in node.names: + counts[item.asname or item.name.split(".")[0]] += 1 + elif isinstance(node, ast.ImportFrom): + for item in node.names: + counts[item.asname or item.name] += 1 + if item.name == "*": + wildcard_import = True + elif isinstance(node, ast.ExceptHandler): + if node.name: + counts[node.name] += 1 + elif isinstance(node, (ast.MatchAs, ast.MatchStar)): + if node.name: + counts[node.name] += 1 + elif isinstance(node, ast.MatchMapping): + if node.rest: + counts[node.rest] += 1 + stack.extend(ast.iter_child_nodes(node)) + return _BindingCounts(counts, wildcard_import) + + +class _BindingCounts: + """The counts/wildcard_import surface _plans reads off _Bindings.""" + + def __init__(self, counts, wildcard_import): + self.counts = counts + self.wildcard_import = wildcard_import + + def __getitem__(self, name): + return self.counts[name] + + +class _UncertainDataName(pynames.PyName): + def __init__(self, original): + self.original = original + + def get_object(self): + return pyobjects.get_unknown() + + def get_definition_location(self): + return self.original.get_definition_location() + + +class _ModulePlanContext: + __slots__ = ( + "tree", + "classes", + "bindings", + "assignments", + "writes", + "parents", + "functions", + "readers_by_attr", + "class_bindings", + "types", + "plain_class", + "plain_rhs", + "enclosing", + "returned_types", + "receiver_names", + "hook_functions", + ) + + def __init__(self, **fields): + for name, value in fields.items(): + setattr(self, name, value) + + +def _plan_context(module): + tree = module.get_ast() + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + bindings = _count_module_bindings(tree) + assignments = defaultdict(list) + # One breadth-first pass yields the node list in ast.walk order and the + # parent map together; the scope and write phases below consume the list + # instead of re-walking the whole tree three more times. + nodes, parents = _walk_with_parents(tree) + seen_class_bindings = {} + + def class_bindings(node): + cached = seen_class_bindings.get(id(node)) + if cached is None: + cached = seen_class_bindings[id(node)] = _class_bindings(node) + return cached + + # Explicit global writes belong to module identity even inside a function. + for scope in nodes: + if not isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + scoped = list(_scope_nodes(scope.body)) + globals_ = { + name + for node in scoped + if isinstance(node, ast.Global) + for name in node.names + } + if not globals_: + continue + for node in scoped: + names = set() + if isinstance(node, ast.Name) and isinstance( + node.ctx, (ast.Store, ast.Del) + ): + names.add(node.id) + elif isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + names.add(node.name) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update( + item.asname + or ( + item.name.split(".")[0] + if isinstance(node, ast.Import) + else item.name + ) + for item in node.names + ) + elif isinstance(node, ast.ExceptHandler) and node.name: + names.add(node.name) + elif isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name: + names.add(node.name) + elif isinstance(node, ast.MatchMapping) and node.rest: + names.add(node.rest) + for name in names & globals_: + bindings.counts[name] += 1 + parent = parents.get(node) + if isinstance(parent, (ast.Assign, ast.AnnAssign, ast.NamedExpr)): + assignments[name].append(parent) + else: + assignments[name].append(node) + + writes = defaultdict(list) + alias_writes = [] + for node in nodes: + if isinstance(node, ast.Assign): + targets = [ + child + for target in node.targets + for child in ast.walk(target) + if isinstance(child, (ast.Name, ast.Attribute)) + and isinstance(child.ctx, ast.Store) + ] + for target in targets: + if isinstance(target, ast.Name) and parents.get(node) is tree: + assignments[target.id].append(node) + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id in classes + ): + writes[target.value.id, target.attr].append(node) + elif isinstance(target, ast.Attribute) and isinstance( + target.value, ast.Name + ): + alias_writes.append((target, node)) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + target = node.target + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id in classes + ): + writes[target.value.id, target.attr].append(node) + elif isinstance(target, ast.Attribute) and isinstance( + target.value, ast.Name + ): + alias_writes.append((target, node)) + + def types(value, seen=()): + if isinstance(value, (ast.Tuple, ast.List)): + return set().union(*(types(item, seen) for item in value.elts)) + if isinstance(value, ast.Starred): + return types(value.value, seen) + if isinstance(value, ast.Call) and isinstance(value.func, ast.Name): + name = value.func.id + elif isinstance(value, ast.Name): + name = value.id + else: + return set() + if name in seen: + return set() + result = {name} if name in classes else set() + for assignment in assignments[name]: + result.update(types(getattr(assignment, "value", None), (*seen, name))) + return result + + def plain_class(name): + node = classes[name] + return ( + bindings.counts[name] == 1 + and not bindings.wildcard_import + and not node.bases + and not node.keywords + and not node.decorator_list + and not (class_bindings(node)[0] & _CLASS_HOOKS) + and not any((name, hook) in writes for hook in _CLASS_HOOKS) + ) + + def plain_rhs(value, write): + if isinstance(value, ast.Name): + candidates = assignments[value.id] + if ( + bindings.counts[value.id] != 1 + or len(candidates) != 1 + or candidates[0].lineno >= write.lineno + ): + return False + value = getattr(candidates[0], "value", None) + return ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and value.func.id in classes + and plain_class(value.func.id) + and not value.args + and not value.keywords + and classes[value.func.id].lineno < write.lineno + ) + + def enclosing(node, kinds): + while node in parents: + node = parents[node] + if isinstance(node, kinds): + return node + return None + + functions = { + node.name: node + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + # Attribute reads grouped by name in one pass; the per-entry reader scan + # below used to re-walk the whole tree for every planned write. + readers_by_attr = collections.defaultdict(list) + for node in nodes: + if isinstance(node, ast.Attribute) and isinstance(node.ctx, ast.Load): + readers_by_attr[node.attr].append(node) + + def returned_types(function, seeds=None, active=()): + if function.name in active: + return set(), True + local_values = defaultdict(list) + unknown_writes = set() + returns = [] + pending = list(function.body) + while pending: + node = pending.pop() + if isinstance(node, ast.Return): + returns.append(node.value) + elif not isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda) + ): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + local_values[target.id].append(node.value) + elif ( + isinstance(node, (ast.AnnAssign, ast.NamedExpr)) + and isinstance(node.target, ast.Name) + and node.value is not None + ): + local_values[node.target.id].append(node.value) + elif isinstance(node, ast.Name) and isinstance( + node.ctx, (ast.Store, ast.Del) + ): + parent = parents.get(node) + if not isinstance( + parent, (ast.Assign, ast.AnnAssign, ast.NamedExpr) + ): + unknown_writes.add(node.id) + elif isinstance(node, ast.ExceptHandler) and node.name: + unknown_writes.add(node.name) + elif isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name: + unknown_writes.add(node.name) + elif isinstance(node, ast.MatchMapping) and node.rest: + unknown_writes.add(node.rest) + pending.extend(ast.iter_child_nodes(node)) + + def facts(value, seen=()): + if value is None or isinstance(value, ast.Constant): + return set(), False + if isinstance(value, ast.Name): + if value.id in seen: + return set(), True + if value.id in local_values or value.id in unknown_writes: + results = [ + facts(expr, (*seen, value.id)) + for expr in local_values[value.id] + ] + if seeds and value.id in seeds: + results.append(seeds[value.id]) + if value.id in unknown_writes: + results.append((set(), True)) + return set().union(*(names for names, _ in results)), any( + unknown for _, unknown in results + ) + if seeds and value.id in seeds: + return seeds[value.id] + names = types(value) + return names, not names + if isinstance(value, ast.Call): + if isinstance(value.func, ast.Name): + name = value.func.id + if name in classes: + return types(value), False + if name in functions and bindings.counts[name] == 1: + callee = functions[name] + positional = callee.args.posonlyargs + callee.args.args + args = positional + callee.args.kwonlyargs + supplied = { + arg.arg: facts(expr, seen) + for arg, expr in zip(positional, value.args) + } + for keyword in value.keywords: + if keyword.arg: + supplied[keyword.arg] = facts(keyword.value, seen) + for arg in args: + supplied.setdefault(arg.arg, (set(), True)) + return returned_types( + callee, supplied, (*active, function.name) + ) + if isinstance(value.func, ast.Attribute) and value.func.attr in { + "__getattribute__", + "__new__", + }: + primary = value.func.value + if ( + isinstance(primary, ast.Call) + and isinstance(primary.func, ast.Name) + and primary.func.id == "super" + and not bindings.counts["super"] + ): + return set(), False + if ( + isinstance(primary, ast.Name) + and primary.id == "object" + and not bindings.counts["object"] + ): + return set(), False + return set(), True + + results = [facts(value) for value in returns] + return set().union(*(names for names, _ in results)), any( + unknown for _, unknown in results + ) + + def receiver_names(value, node): + if isinstance(value, ast.Name) and value.id in classes: + result = {value.id} + result.update(types(value)) + return result + result = types(value) + if isinstance(value, ast.Name) and not result: + function = enclosing(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + cls = enclosing(node, (ast.ClassDef,)) + if function is not None and cls is not None and function.args.args: + if value.id == function.args.args[0].arg: + return {cls.name} + return result + + # Only augment an existing direct class-write plan; do not add alias support. + for target, event in alias_writes: + for owner in types(target.value): + key = (owner, target.attr) + if key in writes: + writes[key].append(event) + + def hook_functions(cls, hook): + def resolve(value, seen=()): + if not isinstance(value, ast.Name) or value.id in seen: + return [None] + if value.id in functions and bindings.counts[value.id] == 1: + return [functions[value.id]] + if bindings.counts[value.id] == 1 and len(assignments[value.id]) == 1: + return resolve( + getattr(assignments[value.id][0], "value", None), (*seen, value.id) + ) + return [None] + + result = [] + for node in _scope_nodes(cls.body): + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == hook + ): + result.append(node) + elif isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == hook + for target in node.targets + ): + result.extend(resolve(node.value)) + elif ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == hook + ): + result.extend(resolve(node.value)) + elif isinstance(node, (ast.Import, ast.ImportFrom)) and any( + (item.asname or item.name) == hook for item in node.names + ): + result.append(None) + for event in writes.get((cls.name, hook), ()): + result.extend(resolve(getattr(event, "value", None))) + if not result and hook in class_bindings(cls)[0]: + result.append(None) + return result + + return _ModulePlanContext( + tree=tree, + classes=classes, + bindings=bindings, + assignments=assignments, + writes=writes, + parents=parents, + functions=functions, + readers_by_attr=readers_by_attr, + class_bindings=class_bindings, + types=types, + plain_class=plain_class, + plain_rhs=plain_rhs, + enclosing=enclosing, + returned_types=returned_types, + receiver_names=receiver_names, + hook_functions=hook_functions, + ) + + +class _ModulePlanState: + """Shared per-module plan store. + + Every (owner, attribute) entry is resolved at most once into ``result`` + and is then shared by all consumers: add_class_data records observation + downgrades (unsupported existing PyName, original values) into the plan + dicts, and the rename checks must observe exactly those records. + """ + + __slots__ = ("context", "result", "complete") + + def __init__(self, context): + self.context = context + self.result = {} + self.complete = False + + +def _plan_state(module): + state = module.__dict__.get("_dc_plan_state") + if state is None: + state = module.__dict__["_dc_plan_state"] = _ModulePlanState( + _plan_context(module) + ) + return state + + +def _resolve_entry(context, result, key, events): + tree = context.tree + classes = context.classes + bindings = context.bindings + assignments = context.assignments + writes = context.writes + parents = context.parents + functions = context.functions + readers_by_attr = context.readers_by_attr + class_bindings = context.class_bindings + types = context.types + plain_class = context.plain_class + plain_rhs = context.plain_rhs + enclosing = context.enclosing + returned_types = context.returned_types + receiver_names = context.receiver_names + hook_functions = context.hook_functions + owner, attribute = key + value_types = set().union( + *(types(getattr(event, "value", None)) for event in events) + ) + if not value_types: + return + # Functions and custom data descriptors retain their existing boundary. + if any( + class_bindings(classes[name])[0] & {"__get__", "__set__", "__delete__"} + for name in value_types + ): + return + possible_owners = {owner} + for assignment in assignments[owner]: + aliases = types(getattr(assignment, "value", None)) + possible_owners.update(aliases or set(classes)) + for decorator in classes[owner].decorator_list: + if isinstance(decorator, ast.Name) and decorator.id in functions: + returned, unknown = returned_types(functions[decorator.id]) + possible_owners.update(set(classes) if unknown else returned) + reasons = [] + event = events[0] + if len(events) != 1: + reasons.append("multiple writes") + if not ( + isinstance(event, ast.Assign) + and len(event.targets) == 1 + and parents.get(event) is tree + ): + reasons.append("conditional or unsupported write") + if not plain_class(owner) or classes[owner].lineno >= event.lineno: + reasons.append("class identity or behavior") + if bindings.wildcard_import: + reasons.append("wildcard import identity") + if any( + any( + isinstance(base, ast.Name) and owner in (types(base) or {base.id}) + for base in node.bases + ) + for node in classes.values() + ): + reasons.append("inheritance") + if not plain_rhs(getattr(event, "value", None), event): + reasons.append("data value identity") + for owner_name in possible_owners: + cls = classes[owner_name] + if attribute in class_bindings(cls)[1]: + reasons.append("existing non-assignment binding") + for child in cls.body: + if isinstance(child, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == attribute + for target in child.targets + ): + value_types.update(types(child.value)) + for node in _scope_nodes(cls.body): + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id == attribute + ): + reasons.append("early class namespace reader") + for node in ast.walk(cls): + if ( + isinstance(node, ast.Attribute) + and node.attr == attribute + and isinstance(node.ctx, ast.Store) + ): + reasons.append("instance write") + assignment = parents.get(node) + value_types.update(types(getattr(assignment, "value", None))) + for node in readers_by_attr.get(attribute, ()): + receivers = receiver_names(node.value, node) + if receivers and not (receivers & possible_owners): + continue + if not receivers: + reasons.append("unknown reader") + if ( + enclosing(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) + is not None + ): + reasons.append("deferred reader") + elif node.lineno <= event.lineno: + reasons.append("early reader") + # Lookup/constructor rewrites can return another local data class. + # They stay unknown; include explicit returns in the mutation guard. + pending_types = list(possible_owners | value_types) + seen_types = set() + while pending_types: + class_name = pending_types.pop() + if class_name in seen_types: + continue + seen_types.add(class_name) + cls = classes[class_name] + rewrite_classes = [cls] + for keyword in cls.keywords: + if ( + keyword.arg == "metaclass" + and isinstance(keyword.value, ast.Name) + and keyword.value.id in classes + ): + rewrite_classes.append(classes[keyword.value.id]) + for rewrite in rewrite_classes: + for hook in _VALUE_HOOKS: + for function in hook_functions(rewrite, hook): + if function is None: + added = set(classes) + else: + seeds = {} + parameters = function.args.posonlyargs + function.args.args + if parameters: + seeds[parameters[0].arg] = ({class_name}, False) + returned, unknown = returned_types(function, seeds) + added = set(classes) if unknown else returned + value_types.update(added) + pending_types.extend(added - seen_types) + for owner_name in possible_owners: + result[owner_name, attribute] = { + "event": event, + "allowed": not reasons, + "reasons": tuple(sorted(set(reasons))), + "affected_types": frozenset(value_types), + "unaliased_import": any( + isinstance(node, (ast.Import, ast.ImportFrom)) + and any( + alias.asname is None + and ( + alias.name.split(".")[0] + if isinstance(node, ast.Import) + else alias.name + ) + == attribute + for alias in node.names + ) + for node in _scope_nodes(classes[owner_name].body) + ), + } + + +def _resolve_owner(state, owner): + result = state.result + for key, events in state.context.writes.items(): + if key[0] != owner or key in result: + continue + _resolve_entry(state.context, result, key, events) + return {key[1]: plan for key, plan in result.items() if key[0] == owner} + + +@utils.saveit +def _plans(module): + state = _plan_state(module) + if not state.complete: + for key, events in state.context.writes.items(): + if key not in state.result: + _resolve_entry(state.context, state.result, key, events) + state.complete = True + return state.result + + +def _plans_for(module, owner): + return _resolve_owner(_plan_state(module), owner) + + +def add_class_data(pyclass, attributes): + module = pyclass.get_module() + if pyclass.parent is not module: + return attributes + for name, plan in _plans_for(module, pyclass.get_name()).items(): + original = attributes.get(name) + if plan["allowed"] and isinstance( + original, (pynames.ImportedName, pynames.ImportedModule) + ): + original = pynamesdef.AssignedName(plan["event"].lineno, module) + if original is None: + original = pynamesdef.AssignedName(plan["event"].lineno, module) + if plan["allowed"] and isinstance(original, pynamesdef.AssignedName): + original.assignments.append(pynamesdef.AssignmentValue(plan["event"].value)) + original.invalidate() + attributes[name] = original + else: + if plan["allowed"]: + plan["allowed"] = False + plan["reasons"] = (*plan["reasons"], "unsupported existing PyName") + originals = plan.setdefault("original_values", []) + if original not in originals: + originals.append(original) + attributes[name] = _UncertainDataName(original) + return attributes + + +def check_rename(module, name, pyname, primary): + modules = [module] + defining, _ = pyname.get_definition_location() + if defining is not None and defining not in modules: + modules.append(defining) + if primary is not None: + obj = primary.get_object() + cls = obj if isinstance(obj, pyobjects.AbstractClass) else obj.get_type() + if isinstance(cls, pyobjects.PyDefinedObject): + owning = cls.get_module() + if owning not in modules: + modules.append(owning) + for current in modules: + _check_module_rename(current, name, pyname) + + +def _module_source(module): + resource = module.get_resource() if hasattr(module, "get_resource") else None + if resource is not None and not resource.is_folder(): + return resource.read() + return getattr(module, "source_code", None) + + +def _check_module_rename(module, name, pyname): + # Plans only exist for attribute writes spelled `.name` somewhere in the + # module; without that text there is nothing this module can contribute. + source = _module_source(module) + if source is None or f".{name}" not in source: + return + plans = _plans(module) + classes = [ + cls + for cls in module._get_defined_objects() or () + if isinstance(cls, pyobjects.AbstractClass) and cls.parent is module + ] + planned_owners = {owner for owner, _ in plans} + # Resolve unsupported existing PyNames before deciding the cold guard. + for cls in classes: + if cls.get_name() in planned_owners: + cls._get_structural_attributes() + for cls in classes: + plan = plans.get((cls.get_name(), name)) + if ( + plan is not None + and plan.get("unaliased_import") + and cls.get_attributes().get(name) is pyname + ): + raise exceptions.RefactoringError( + "Cannot rename this import-backed class attribute without an explicit import alias." + ) + blocked = {key: plan for key, plan in plans.items() if not plan["allowed"]} + if not blocked: + return + affected = set().union(*(plan["affected_types"] for plan in blocked.values())) + for cls in classes: + if ( + cls.get_name() in affected or (cls.get_name(), name) in blocked + ) and cls.get_attributes().get(name) is pyname: + raise exceptions.RefactoringError( + "Cannot rename this field while external class data has uncertain identity or execution order." + ) + for plan in blocked.values(): + for container in _original_value_containers(plan): + if container.get_attributes().get(name) is pyname: + raise exceptions.RefactoringError( + "Cannot rename this field while the original class data value has uncertain uses." + ) + + +def _original_value_containers(plan): + # Keep PyNames: each guard observes the current imported module cache. + seen_modules = set() + seen_classes = set() + + def local_source_module(module, project, root): + if ( + not isinstance(module, pyobjects.AbstractModule) + or not isinstance(module, pyobjects.PyDefinedObject) + or root is None + ): + return False + resource = module.get_resource() + return ( + resource is not None + and resource.project is project + and (resource == root or root.contains(resource)) + and resource.pathlib.resolve().is_relative_to(root.pathlib.resolve()) + ) + + for original in plan.get("original_values", ()): + obj = original.get_object() + if obj is pyobjects.get_unknown(): + continue + container = ( + obj + if isinstance(obj, (pyobjects.AbstractClass, pyobjects.AbstractModule)) + else obj.get_type() + ) + resource = ( + container.get_module().get_resource() + if isinstance( + container, (pyobjects.AbstractModule, pyobjects.AbstractClass) + ) + and isinstance(container, pyobjects.PyDefinedObject) + else None + ) + project = resource.project if resource is not None else None + root = getattr(project, "root", None) + pending = [container] + while pending: + current = pending.pop() + if isinstance(current, pyobjects.AbstractClass): + if id(current) in seen_classes: + continue + seen_classes.add(id(current)) + yield current + if isinstance( + current, pyobjects.PyDefinedObject + ) and local_source_module(current.get_module(), project, root): + for defined in current._get_defined_objects() or (): + if ( + isinstance(defined, pyobjects.AbstractClass) + and defined.parent is current + ): + pending.append(defined) + continue + if not isinstance(current, pyobjects.AbstractModule): + continue + if id(current) in seen_modules: + continue + seen_modules.add(id(current)) + yield current + if not isinstance(current, pyobjects.PyDefinedObject): + continue + # Preserve the original direct-class guard for dynamic module access. + for defined in current._get_defined_objects() or (): + if ( + isinstance(defined, pyobjects.AbstractClass) + and defined.parent is current + ): + pending.append(defined) + if not local_source_module(current, project, root): + continue + # Follow existing inferred exports, never arbitrary class attributes. + for exported in current.get_attributes().values(): + value = exported.get_object() + if value is pyobjects.get_unknown(): + continue + if isinstance(exported, pynames.ImportedName) and isinstance( + value, pyobjects.AbstractFunction + ): + defining, _ = exported.get_definition_location() + if local_source_module(defining, project, root): + pending.append(defining) + candidate = ( + value + if isinstance( + value, (pyobjects.AbstractClass, pyobjects.AbstractModule) + ) + else value.get_type() + ) + if isinstance(candidate, pyobjects.AbstractClass): + pending.append(candidate) + elif local_source_module(candidate, project, root): + pending.append(candidate) + + +def import_source_name( + scope, + lineno, + offset, + word, + primary, + from_module, + from_aliased, + from_is_module, +): + """Preserve import source identity only for class bindings touched by this layer.""" + if ( + scope.get_kind() != "Class" + or not isinstance(scope.pyobject, pyobjects.PyDefinedObject) + or scope.pyobject.parent is not scope.pyobject.get_module() + ): + return None + module = scope.pyobject.get_module() + plans = _plans(module) + owner = scope.pyobject.get_name() + for node in _scope_nodes(scope.pyobject.get_ast().body): + if ( + not isinstance(node, (ast.Import, ast.ImportFrom)) + or not node.lineno <= lineno <= node.end_lineno + ): + continue + for alias in node.names: + local = alias.asname or ( + alias.name.split(".")[0] if isinstance(node, ast.Import) else alias.name + ) + if (owner, local) not in plans: + continue + if isinstance(node, ast.ImportFrom): + origin = "." * node.level + (node.module or "") + if from_module != origin: + continue + imported = pynamesdef.ImportedModule( + module, node.module or "", node.level + ) + if from_is_module: + level = len(primary) - len(primary.lstrip(".")) + return pynamesdef.ImportedModule(module, primary.lstrip("."), level) + if alias.name == word and (alias.asname is None or from_aliased): + return pynamesdef.ImportedName(imported, alias.name) + elif alias.name == primary or alias.name.startswith(primary + "."): + start, end = codeanalyze.ASTLinesAdapter(module.source_code)[alias] + if start is None or end is None: + continue + origin_end = end + if alias.asname is not None: + segment = module.source_code[start:end] + for token in tokenize.generate_tokens( + io.StringIO(segment).readline + ): + if token.type == tokenize.NAME and token.string == "as": + lines = segment.split("\n") + origin_end = ( + start + + sum( + len(line) + 1 + for line in lines[: token.start[0] - 1] + ) + + token.start[1] + ) + break + if start <= offset < origin_end: + return pynamesdef.ImportedModule(module, primary) + return None diff --git a/rope/base/fscommands.py b/rope/base/fscommands.py index 36514312c..e80fe25af 100644 --- a/rope/base/fscommands.py +++ b/rope/base/fscommands.py @@ -35,6 +35,16 @@ def create_fscommands(root): class FileSystemCommands: + exists = staticmethod(os.path.exists) + isfile = staticmethod(os.path.isfile) + isdir = staticmethod(os.path.isdir) + listdir = staticmethod(os.listdir) + stat = staticmethod(os.stat) + + @staticmethod + def realpath(path): + return os.path.realpath(os.path.expanduser(path)) + def create_file(self, path): open(path, "w").close() @@ -62,7 +72,7 @@ def read(self, path): return handle.read() -class SubversionCommands: +class SubversionCommands(FileSystemCommands): def __init__(self, *args): self.normal_actions = FileSystemCommands() import pysvn # type: ignore @@ -90,7 +100,7 @@ def read(self, path): return self.normal_actions.read(path) -class MercurialCommands: +class MercurialCommands(FileSystemCommands): def __init__(self, root): self.hg = self._import_mercurial() self.normal_actions = FileSystemCommands() @@ -141,7 +151,7 @@ def read(self, path): return self.normal_actions.read(path) -class GITCommands: +class GITCommands(FileSystemCommands): def __init__(self, root): self.root = root self._do(["version"]) @@ -176,7 +186,7 @@ def _in_dir(self, path): return self.root -class DarcsCommands: +class DarcsCommands(FileSystemCommands): def __init__(self, root): self.root = root self.normal_actions = FileSystemCommands() @@ -268,8 +278,7 @@ def read_str_coding(source): for line in source.split(newline, 2)[:2]: if re.match(CODING_LINE_PATTERN, line): return _find_coding(line) - else: - return + return def _find_coding(text): diff --git a/rope/base/libutils.py b/rope/base/libutils.py index 0974b11e7..79ff5ad6a 100644 --- a/rope/base/libutils.py +++ b/rope/base/libutils.py @@ -32,11 +32,15 @@ def path_to_resource(project, path, type=None): def path_relative_to_project_root(project, path): - return relative(project.address, path) + return _relative(project.address, path) @utils.deprecated() def relative(root, path): + return _relative(root, path) + + +def _relative(root, path): root = rope.base.project._realpath(root).replace(os.path.sep, "/") path = rope.base.project._realpath(path).replace(os.path.sep, "/") if path == root: @@ -120,4 +124,18 @@ def modname(resource): module_name = source_folder.name + "." + module_name source_folder = source_folder.parent + # An explicitly elected import root also owns intervening PEP 420 parts. + # Inferred folders retain the historical regular-package naming contract. + declared = { + resource.project.get_folder(path) + for path in resource.project.prefs.get("source_folders", []) + } + ancestor = source_folder + while ancestor not in declared and ancestor != ancestor.parent: + ancestor = ancestor.parent + if ancestor in declared: + while source_folder != ancestor: + module_name = source_folder.name + "." + module_name + source_folder = source_folder.parent + return module_name diff --git a/rope/base/oi/objectinfo.py b/rope/base/oi/objectinfo.py index c6457044b..430921bec 100644 --- a/rope/base/oi/objectinfo.py +++ b/rope/base/oi/objectinfo.py @@ -1,8 +1,29 @@ +import json import warnings -from rope.base import exceptions, resourceobserver +from rope.base import exceptions, pyobjects, resourceobserver from rope.base.oi import memorydb, objectdb, transform +_CALL_LAYOUT_PREFIX = "!parameter-layout-v1:" + + +def _is_layout_key(key): + return isinstance(key, str) and key.startswith(_CALL_LAYOUT_PREFIX) + + +def _call_scope_key(pyfunction, key): + if isinstance(pyfunction, pyobjects.PyFunction): + layout = pyfunction.get_parameter_layout() + kind = pyfunction.get_kind() + if kind != "function" or any( + parameter_kind in ("posonly", "kwonly") for parameter_kind, _ in layout + ): + # Scope keys also appear as JSON object keys in the persisted database. + return _CALL_LAYOUT_PREFIX + json.dumps( + (key, kind, layout), separators=(",", ":") + ) + return key + class ObjectInfoManager: """Stores object information @@ -71,10 +92,14 @@ def get_returned(self, pyobject, args): result = self.get_exact_returned(pyobject, args) if result is not None: return result - path, key = self._get_scope(pyobject) + path, key = self._get_call_scope(pyobject) if path is None: return None for call_info in self.objectdb.get_callinfos(path, key): + if _is_layout_key(key) and len(call_info.get_parameters()) != len( + pyobject.get_param_names(special_args=False) + ): + continue returned = call_info.get_returned() if returned and returned[0] not in ("unknown", "none"): result = returned @@ -85,8 +110,10 @@ def get_returned(self, pyobject, args): return self.to_pyobject(result) def get_exact_returned(self, pyobject, args): - path, key = self._get_scope(pyobject) + path, key = self._get_call_scope(pyobject) if path is not None: + if args is None: + return None returned = self.objectdb.get_returned( path, key, self._args_to_textual(pyobject, args) ) @@ -100,7 +127,7 @@ def _args_to_textual(self, pyfunction, args): return textual_args def get_parameter_objects(self, pyobject): - path, key = self._get_scope(pyobject) + path, key = self._get_call_scope(pyobject) if path is None: return None arg_count = len(pyobject.get_param_names(special_args=False)) @@ -108,6 +135,8 @@ def get_parameter_objects(self, pyobject): parameters = [None] * arg_count for call_info in self.objectdb.get_callinfos(path, key): args = call_info.get_parameters() + if _is_layout_key(key) and len(args) != arg_count: + continue for index, arg in enumerate(args[:arg_count]): old = parameters[index] if self.validation.is_more_valid(arg, old): @@ -120,12 +149,17 @@ def get_parameter_objects(self, pyobject): return [self.to_pyobject(parameter) for parameter in parameters] def get_passed_objects(self, pyfunction, parameter_index): - path, key = self._get_scope(pyfunction) + path, key = self._get_call_scope(pyfunction) if path is None: return [] result = [] + count = len(pyfunction.get_param_names(special_args=False)) + if parameter_index >= count: + return result for call_info in self.objectdb.get_callinfos(path, key): args = call_info.get_parameters() + if _is_layout_key(key) and len(args) != count: + continue if len(args) > parameter_index: parameter = self.to_pyobject(args[parameter_index]) if parameter is not None: @@ -145,11 +179,12 @@ def doi_to_normal(textual): def function_called(self, pyfunction, params, returned=None): function_text = self.to_textual(pyfunction) - params_text = tuple(self.to_textual(param) for param in params) + count = len(pyfunction.get_param_names(special_args=False)) + params_text = tuple(self.to_textual(param) for param in params[:count]) returned_text = ("unknown",) if returned is not None: returned_text = self.to_textual(returned) - self._save_data(function_text, params_text, returned_text) + self._save_data(function_text, params_text, returned_text, pyfunction) def save_per_name(self, scope, name, data): path, key = self._get_scope(scope.pyobject) @@ -163,8 +198,26 @@ def get_per_name(self, scope, name): if result is not None: return self.to_pyobject(result) - def _save_data(self, function, args, returned=("unknown",)): - self.objectdb.add_callinfo(function[1], function[2], args, returned) + def _save_data(self, function, args, returned=("unknown",), pyfunction=None): + if pyfunction is None: + # DOA data arrives as text; the live inference path always passes + # the function itself because resolving the text back through + # class attributes can re-enter the very inference in progress + # (e.g. a property getter records its own call information). + pyfunction = self.to_pyobject(function) + if not isinstance(pyfunction, pyobjects.PyFunction): + return + path, key = self._get_call_scope(pyfunction) + count = len(pyfunction.get_param_names(special_args=False)) + if path is None or (_is_layout_key(key) and len(args) < count): + return + self.objectdb.add_callinfo(path, key, args[:count], returned) + + def _get_call_scope(self, pyfunction): + path, key = self._get_scope(pyfunction) + if path is not None: + key = _call_scope_key(pyfunction, key) + return path, key def _get_scope(self, pyobject): resource = pyobject.get_module().get_resource() @@ -206,11 +259,43 @@ def is_file_valid(self, path): return self.to_pyobject.path_to_resource(path) is not None def is_scope_valid(self, path, key): + call_key = None + if _is_layout_key(key): + call_key = key + try: + encoded = json.loads(key[len(_CALL_LAYOUT_PREFIX) :]) + except (ValueError, TypeError): + return False + if not ( + isinstance(encoded, list) + and len(encoded) == 3 + and isinstance(encoded[0], str) + and encoded[1] in ("function", "method", "staticmethod", "classmethod") + and isinstance(encoded[2], list) + and all( + isinstance(parameter, list) + and len(parameter) == 2 + and parameter[0] + in ("posonly", "positional", "kwonly", "vararg", "kwarg") + and isinstance(parameter[1], str) + for parameter in encoded[2] + ) + ): + return False + key = encoded[0] + elif not isinstance(key, str): + return False if key == "": textual = ("defined", path) else: textual = ("defined", path, key) - return self.to_pyobject(textual) is not None + pyobject = self.to_pyobject(textual) + if call_key is not None: + return ( + isinstance(pyobject, pyobjects.PyFunction) + and _call_scope_key(pyobject, key) == call_key + ) + return pyobject is not None class _FileListObserver: diff --git a/rope/base/oi/runmod.py b/rope/base/oi/runmod.py index e7219b5f9..395313070 100644 --- a/rope/base/oi/runmod.py +++ b/rope/base/oi/runmod.py @@ -84,7 +84,9 @@ def on_function_call(self, frame, event, arg): args = [] returned = ("unknown",) code = frame.f_code - for argname in code.co_varnames[: code.co_argcount]: + for argname in code.co_varnames[ + : code.co_argcount + code.co_kwonlyargcount + ]: try: argvalue = self._object_to_persisted_form(frame.f_locals[argname]) args.append(argvalue) diff --git a/rope/base/oi/soa.py b/rope/base/oi/soa.py index df60c6fbb..df585a1f7 100644 --- a/rope/base/oi/soa.py +++ b/rope/base/oi/soa.py @@ -1,24 +1,46 @@ import rope.base.ast import rope.base.oi.soi import rope.base.pynames -from rope.base import arguments, evaluate, nameanalyze, pyobjects +from rope.base import arguments, classfunctions, evaluate, nameanalyze, pyobjects -def analyze_module(pycore, pymodule, should_analyze, search_subscopes, followed_calls): +def analyze_module( + pycore, + pymodule, + should_analyze, + search_subscopes, + followed_calls, + *, + visitor_factory=None, +): """Analyze `pymodule` for static object inference Analyzes scopes for collecting object information. The analysis starts from inner scopes. """ - _analyze_node(pycore, pymodule, should_analyze, search_subscopes, followed_calls) + _analyze_node( + pycore, + pymodule, + should_analyze, + search_subscopes, + followed_calls, + visitor_factory, + ) -def _analyze_node(pycore, pydefined, should_analyze, search_subscopes, followed_calls): +def _analyze_node( + pycore, pydefined, should_analyze, search_subscopes, followed_calls, visitor_factory +): if search_subscopes(pydefined): for scope in pydefined.get_scope().get_scopes(): _analyze_node( - pycore, scope.pyobject, should_analyze, search_subscopes, followed_calls + pycore, + scope.pyobject, + should_analyze, + search_subscopes, + followed_calls, + visitor_factory, ) if should_analyze(pydefined): new_followed_calls = max(0, followed_calls - 1) @@ -27,10 +49,16 @@ def _analyze_node(pycore, pydefined, should_analyze, search_subscopes, followed_ def _follow(pyfunction): _analyze_node( - pycore, pyfunction, return_true, return_false, new_followed_calls + pycore, + pyfunction, + return_true, + return_false, + new_followed_calls, + visitor_factory, ) - visitor = SOAVisitor(pycore, pydefined, _follow if followed_calls else None) + factory = SOAVisitor if visitor_factory is None else visitor_factory + visitor = factory(pycore, pydefined, _follow if followed_calls else None) for child in rope.base.ast.iter_child_nodes(pydefined.get_ast()): visitor.visit(child) @@ -71,15 +99,20 @@ def _Call(self, node): self._call(pyfunction, args) def _args_with_self(self, primary, self_pyname, pyfunction, node): - base_args = arguments.create_arguments(primary, pyfunction, node, self.scope) + base_args = arguments.create_arguments( + primary, pyfunction, node, self.scope, ignore_instance=True + ) return arguments.MixedArguments(self_pyname, base_args, self.scope) def _call(self, pyfunction, args): + if isinstance(pyfunction, classfunctions.BoundFunction): + args = pyfunction.bind_arguments(args) + pyfunction = pyfunction.function if isinstance(pyfunction, pyobjects.PyFunction): if self.follow is not None: before = self._parameter_objects(pyfunction) self.pycore.object_info.function_called( - pyfunction, args.get_arguments(pyfunction.get_param_names()) + pyfunction, args.get_arguments(pyfunction.get_param_names(False)) ) pyfunction._set_parameter_pyobjects(None) if self.follow is not None: @@ -117,10 +150,155 @@ def _Assign(self, node): nodes.extend(visitor.nodes) self._evaluate_assign_value(node, nodes) - def _evaluate_assign_value(self, node, nodes, type_hint=False): - for subscript, levels in nodes: - instance = evaluate.eval_node(self.scope, subscript.value) - args_pynames = [evaluate.eval_node(self.scope, subscript.slice)] + def _operator_effects(self, node, left, right, *, inplace=False): + for child in rope.base.ast.iter_child_nodes(node): + self.visit(child) + operation = { + rope.base.ast.Add: "add", + rope.base.ast.Sub: "sub", + rope.base.ast.Mult: "mul", + rope.base.ast.MatMult: "matmul", + rope.base.ast.Div: "truediv", + rope.base.ast.FloorDiv: "floordiv", + rope.base.ast.Mod: "mod", + rope.base.ast.Pow: "pow", + rope.base.ast.LShift: "lshift", + rope.base.ast.RShift: "rshift", + rope.base.ast.BitOr: "or", + rope.base.ast.BitXor: "xor", + rope.base.ast.BitAnd: "and", + }[type(node.op)] + left = evaluate.eval_node(self.scope, left) + right = evaluate.eval_node(self.scope, right) + effects = [ + (left, right, "__" + operation + "__"), + (right, left, "__r" + operation + "__"), + ] + if self._reflected_first(left, right, effects[1][2]): + effects.reverse() + if inplace: + effects.insert(0, (left, right, "__i" + operation + "__")) + # Python tries each method in order and falls back only when one is + # missing or returns NotImplemented. + for receiver, operand, name in effects: + if receiver is None or name not in receiver.get_object(): + continue + result = self._protocol_call(receiver, name, [operand]) + if not self._may_be_not_implemented(result): + break + + @staticmethod + def _reflected_first(left, right, name): + """A right operand of a proper subclass overriding the method goes first.""" + if left is None or right is None: + return False + left_type = left.get_object().get_type() + right_type = right.get_object().get_type() + if ( + not isinstance(left_type, pyobjects.PyClass) + or not isinstance(right_type, pyobjects.PyClass) + or left_type is right_type + ): + return False + from rope.base.pyobjectsdef import _inheritance_order + + if left_type not in _inheritance_order(right_type): + return False + reflected = right_type.get_attributes().get(name) + inherited = left_type.get_attributes().get(name) + return reflected is not None and ( + inherited is None or reflected.get_object() is not inherited.get_object() + ) + + @staticmethod + def _may_be_not_implemented(result): + if result is None: + return True + value = result.get_object() + if value is None or value == pyobjects.get_unknown(): + return True + return value.get_type() is rope.base.builtins._runtime_class( + type(NotImplemented) + ) + + def _AugAssign(self, node): + self._operator_effects(node, node.target, node.value, inplace=True) + + def _BinOp(self, node): + self._operator_effects(node, node.left, node.right) + + def _protocol_call(self, receiver, name, operands=()): + if receiver is None or name not in receiver.get_object(): + return None + function = receiver.get_object()[name].get_object() + args = arguments.ObjectArguments([receiver, *operands]) + self._call(function, args) + if isinstance(function, pyobjects.AbstractFunction): + result = function.get_returned_object(args) + if result is not None: + return rope.base.pynames.UnboundName(result) + + def _Compare(self, node): + for child in rope.base.ast.iter_child_nodes(node): + self.visit(child) + left = evaluate.eval_node(self.scope, node.left) + for operation, expression in zip(node.ops, node.comparators): + right = evaluate.eval_node(self.scope, expression) + if isinstance(operation, (rope.base.ast.In, rope.base.ast.NotIn)): + if right is not None and "__contains__" in right.get_object(): + self._protocol_call(right, "__contains__", [left]) + else: + iterator = self._protocol_call(right, "__iter__") + if iterator is not None: + # Rope's builtin iterators model `next`, not `__next__`. + item = self._protocol_call( + iterator, + ( + "__next__" + if "__next__" in iterator.get_object() + else "next" + ), + ) + else: + item = self._protocol_call( + right, + "__getitem__", + [ + rope.base.pynames.UnboundName( + pyobjects.PyObject( + rope.base.builtins.builtins["int"].get_object() + ) + ) + ], + ) + self._protocol_call(item, "__eq__", [left]) + self._protocol_call(left, "__eq__", [item]) + else: + names = { + rope.base.ast.Eq: ("__eq__", "__eq__"), + rope.base.ast.NotEq: ("__ne__", "__ne__"), + rope.base.ast.Lt: ("__lt__", "__gt__"), + rope.base.ast.LtE: ("__le__", "__ge__"), + rope.base.ast.Gt: ("__gt__", "__lt__"), + rope.base.ast.GtE: ("__ge__", "__le__"), + }.get(type(operation)) + if names is not None: + self._protocol_call(left, names[0], [right]) + self._protocol_call(right, names[1], [left]) + left = right + + def _evaluate_assign_value(self, node, nodes, type_hint=None): + for target, levels in nodes: + instance = evaluate.eval_node(self.scope, target.value) + if isinstance(target, rope.base.ast.Attribute): + operation = "__setattr__" + key = rope.base.pynames.UnboundName( + rope.base.builtins.LiteralString(target.attr) + ) + else: + operation = "__setitem__" + key = evaluate.eval_node(self.scope, target.slice) + args_pynames = [key] value = rope.base.oi.soi._infer_assignment( rope.base.pynames.AssignmentValue( node.value, levels, type_hint=type_hint @@ -130,8 +308,13 @@ def _evaluate_assign_value(self, node, nodes, type_hint=False): args_pynames.append(rope.base.pynames.UnboundName(value)) if instance is not None and value is not None: pyobject = instance.get_object() - if "__setitem__" in pyobject: - pyfunction = pyobject["__setitem__"].get_object() + if operation == "__setattr__" and isinstance( + pyobject, pyobjects.AbstractClass + ): + # Assigning a class attribute calls the metaclass method. + pyobject = pyobject.get_metaclass() + if pyobject is not None and operation in pyobject: + pyfunction = pyobject[operation].get_object() args = arguments.ObjectArguments([instance] + args_pynames) self._call(pyfunction, args) # IDEA: handle `__setslice__`, too @@ -143,7 +326,8 @@ def __init__(self): self.nodes = [] def _added(self, node, levels): - if isinstance(node, rope.base.ast.Subscript) and isinstance( - node.slice, (rope.base.ast.Index, rope.base.ast.expr) + if isinstance(node, rope.base.ast.Attribute) or ( + isinstance(node, rope.base.ast.Subscript) + and isinstance(node.slice, (rope.base.ast.Index, rope.base.ast.expr)) ): self.nodes.append((node, levels)) diff --git a/rope/base/oi/soi.py b/rope/base/oi/soi.py index 090b10337..c81709e71 100644 --- a/rope/base/oi/soi.py +++ b/rope/base/oi/soi.py @@ -5,9 +5,12 @@ """ +import typing + import rope.base.builtins # Use full qualification for clarity. -from rope.base import arguments, evaluate, pynames, pyobjects, utils +from rope.base import arguments, ast, evaluate, pynames, pyobjects, utils from rope.base.oi.type_hinting.factory import get_type_hinting_factory +from rope.base.oi.type_hinting.utils import hint_instance _ignore_inferred = utils.ignore_exception(pyobjects.IsBeingInferredError) @@ -33,7 +36,7 @@ def infer_returned_object(pyfunction, args): ).make_return_provider() type_ = hint_return(pyfunction) if type_ is not None: - return pyobjects.PyObject(type_) + return hint_instance(type_) @_ignore_inferred @@ -49,41 +52,268 @@ def infer_parameter_objects(pyfunction): def _handle_first_parameter(pyobject, parameters): kind = pyobject.get_kind() + if not pyobject.get_positional_param_names(): + return if not parameters: - if not pyobject.get_param_names(special_args=False): - return parameters.append(pyobjects.get_unknown()) + if parameters[0] is not None and parameters[0] != pyobjects.get_unknown(): + return if kind == "method": parameters[0] = pyobjects.PyObject(pyobject.parent) if kind == "classmethod": parameters[0] = pyobject.parent +def relevant_branch_context(pyname, branch_context): + """Return the part of `branch_context` that selects `pyname` assignments + + Branches that contain none of the assignments of `pyname` cannot change + its value and are dropped. + + """ + relevant = set() + for assignment in pyname.assignments: + assignment_context = _assignment_branch_context(assignment, pyname.module) + if assignment_context is not None: + relevant.update(branch for branch, _ in assignment_context) + return tuple((branch, arm) for branch, arm in branch_context if branch in relevant) + + @_ignore_inferred -def infer_assigned_object(pyname): - if not pyname.assignments: - return - for assignment in reversed(pyname.assignments): - result = _infer_assignment(assignment, pyname.module) +def infer_assigned_object(pyname, *, branch_context=(), read_node=None): + read_point = None + if pyname.initial_binding is not None: + source_index = pyname.module._get_source_binding_index() + read_context = source_index.context_for_node(read_node) if ( + read_context is None + or read_context.structural_scope.pyobject + is not pyname.initial_binding.pyfunction + ): + return pyobjects.get_unknown() + read_point = source_index._span(read_node)[0] + elif read_node is not None and read_node is pyname.module.get_ast(): + # A read after the whole module executed (PEP 649 lazy annotations): + # every assignment is visible and each value keeps its own epoch. + read_point = pyname.module._get_source_binding_index()._span(read_node)[1] + elif read_node is not None: + source_index = pyname.module._get_source_binding_index() + read_context = source_index.context_for_node(read_node) + if read_context is not None: + read_point = source_index._span(read_node)[0] + context = dict(relevant_branch_context(pyname, branch_context)) + context.update(pyname._inference_assignment_context) + limit = pyname._inference_assignment_limit + if limit is None: + limit = len(pyname.assignments) + if not limit: + if any(arm is None for arm in context.values()): + pyname._inference_assignment_ambiguous = True + return pyobjects.get_unknown() + if pyname.initial_binding is not None: + return pyname.initial_binding.get_object() + return + result, ambiguous = _infer_assignment_path( + pyname, + limit - 1, + context, + pyname._inference_assignment_limit is not None or read_point is not None, + read_point, + ) + if ambiguous: + if pyname._inference_assignment_limit is not None: + pyname._inference_assignment_ambiguous = True + return pyobjects.get_unknown() + if ( + result is not None + and result is not pyobjects.get_unknown() + and not ( isinstance(result, rope.base.builtins.BuiltinUnknown) and result.get_name() == "NotImplementedType" - ): - break - elif result == pyobjects.get_unknown(): - break - elif result is not None: - return result + ) + ): + return result hint_assignment = get_type_hinting_factory( pyname.module.pycore.project ).make_assignment_provider() hinting_result = hint_assignment(pyname) if hinting_result is not None: - return pyobjects.PyObject(hinting_result) + return hint_instance(hinting_result) return result +def _infer_assignment_path(pyname, index, context, recursive, read_point=None): + if recursive and any(arm is None for arm in context.values()): + return pyobjects.get_unknown(), True + while index >= 0: + assignment = pyname.assignments[index] + if read_point is not None and assignment.ast_node is None: + index -= 1 + continue + branch_context = _assignment_branch_context(assignment, pyname.module) + if branch_context is None: + return pyobjects.get_unknown(), True + if read_point is not None: + source_index = pyname.module._get_source_binding_index() + if source_index.context_for_node(assignment.ast_node) is None: + return pyobjects.get_unknown(), True + if source_index._span(assignment.ast_node)[1] > read_point: + index -= 1 + continue + for branch, arm in branch_context: + if arm is None: + continue + if isinstance(branch.test, ast.Constant): + selected = bool(branch.test.value) + else: + selected = context.get(branch) + if selected is None and _is_type_checking(branch.test, pyname.module): + selected = True + if selected is not None and selected != arm: + break + if selected is None: + matching = dict(context) + matching[branch] = arm + other = dict(context) + other[branch] = not arm + first, first_ambiguous = _infer_assignment_path( + pyname, index, matching, recursive, read_point + ) + second, second_ambiguous = _infer_assignment_path( + pyname, index - 1, other, recursive, read_point + ) + if not first_ambiguous and not second_ambiguous and first is second: + return first, False + return pyobjects.get_unknown(), True + else: + unsupported = {branch: arm for branch, arm in branch_context if arm is None} + if recursive and unsupported: + return pyobjects.get_unknown(), True + previous_limit = pyname._inference_assignment_limit + previous_context = pyname._inference_assignment_context + previous_ambiguous = pyname._inference_assignment_ambiguous + pyname._inference_assignment_limit = index + pyname._inference_assignment_context = tuple(context.items()) + tuple( + unsupported.items() + ) + pyname._inference_assignment_ambiguous = False + try: + result = _infer_assignment( + assignment, + pyname.module, + branch_context=tuple(context.items()) + branch_context, + source_epoch=read_point is not None + and pyname.initial_binding is None, + ) + ambiguous = pyname._inference_assignment_ambiguous + finally: + pyname._inference_assignment_limit = previous_limit + pyname._inference_assignment_context = previous_context + pyname._inference_assignment_ambiguous = previous_ambiguous + if ambiguous: + return pyobjects.get_unknown(), True + if result is not None: + return result, False + if read_point is not None: + return pyobjects.get_unknown(), True + index -= 1 + if pyname.initial_binding is not None: + return pyname.initial_binding.get_object(), False + return None, False + + +def _is_type_checking(test, pymodule): + """Tell whether a branch test reads `typing.TYPE_CHECKING` + + PEP 484 has static analysis assume `typing.TYPE_CHECKING` is true. The + decision is kept with the module's concluded data, so it is made once + until a module of the project changes. + + """ + decided = pymodule._type_checking_tests() + if decided.get() is None: + decided.set({}) + if test not in decided.get(): + decided.get()[test] = _reads_type_checking(test, pymodule) + return decided.get()[test] + + +def _reads_type_checking(test, pymodule): + """Follow the bindings a branch test reads to `typing.TYPE_CHECKING` + + Imports, attributes of imported modules and a name's single plain + assignment (typing_extensions binds ``TYPE_CHECKING = + typing.TYPE_CHECKING``) are followed, each binding at most once. No + object is inferred: a test that needs inference to resolve, a name + spelled alike elsewhere, a name written more than once (rebound, + augmented, deleted, or assigned under ``global``/``nonlocal``) and a + `typing` module without the constant all read as not `TYPE_CHECKING`. + + The constant is recognized by its place, the ``TYPE_CHECKING`` attribute + of the module named `typing`, not by its value: a project ``typing.py`` + or ``typing/__init__.py`` that shadows the standard module and binds + ``TYPE_CHECKING = False`` is still read as true. + + """ + pyname, name = _read_binding(test, pymodule) + followed = set() + while pyname is not None and pyname not in followed: + followed.add(pyname) + if isinstance(pyname, pynames.ImportedName): + name = pyname.imported_name + pyname = pyname._get_imported_pyname() + continue + if not isinstance(pyname, pynames.AssignedName): + return False + module, _ = pyname.get_definition_location() + if module is None: + return False + if ( + module.get_attributes().get("TYPE_CHECKING") is pyname + and module.get_name() == "typing" + ): + return True + if len(pyname.assignments) != 1: + return False + if len(module._get_source_binding_index().name_writes(pyname, name)) != 1: + return False + (assignment,) = pyname.assignments + if assignment.levels or assignment.evaluation: + return False + pyname, name = _read_binding(assignment.ast_node, module) + return False + + +def _read_binding(node, pymodule): + """Return the binding a name or an imported module's attribute reads + + The binding comes with the name it is bound to in its own namespace. + """ + if isinstance(node, ast.Name): + context = pymodule._get_source_binding_index().context_for_node(node) + if context is not None: + return context.environment.lookup(node.id), node.id + if isinstance(node, ast.Attribute): + owner, _ = _read_binding(node.value, pymodule) + if isinstance(owner, pynames.ImportedModule): + return owner.get_object().get_attributes().get(node.attr), node.attr + return None, None + + +def _assignment_branch_context(assignment, pymodule): + node = assignment.type_hint or assignment.ast_node + context = pymodule._get_source_binding_index().context_for_node(node) + if context is not None: + return context.branch_context + if assignment.branch_context is not None: + return assignment.branch_context + # Assignments outside the source binding index (for instance those of an + # independently parsed expression) carry no branch information; like + # upstream, they are inferred as unconditional assignments. + return () + + def get_passed_objects(pyfunction, parameter_index): object_info = pyfunction.pycore.object_info result = object_info.get_passed_objects(pyfunction, parameter_index) @@ -130,7 +360,7 @@ def _parameter_objects(pyobject): for name in params: type_ = hint_param(pyobject, name) if type_ is not None: - result.append(pyobjects.PyObject(type_)) + result.append(hint_instance(type_)) else: result.append(pyobjects.get_unknown()) return result @@ -139,9 +369,39 @@ def _parameter_objects(pyobject): # handling `rope.base.pynames.AssignmentValue` +def _declared_alias_value(assignment, pymodule): + """Bind a PEP 613 ``X: TypeAlias = Y`` to ``Y`` itself, as ``X = Y`` does. + + The annotation declares an alias, not the type of an instance, so the + assignment is inferred from its value once the annotation resolves to the + runtime ``typing.TypeAlias`` form (``typing_extensions`` re-exports it). + """ + hint = assignment.type_hint + if hint is None or assignment.ast_node is None: + return assignment + scope = pymodule.get_scope().get_inner_scope_for_line(_get_lineno_for_node(hint)) + form = evaluate.eval_node(scope, hint) + if form is None or not rope.base.builtins._is_runtime_form( + form.get_object(), typing.TypeAlias + ): + return assignment + return pynames.AssignmentValue( + assignment.ast_node, + assignment.levels, + assignment.evaluation, + branch_context=assignment.branch_context, + ) + + @_ignore_inferred -def _infer_assignment(assignment, pymodule): - result = _follow_pyname(assignment, pymodule) +def _infer_assignment(assignment, pymodule, *, branch_context=(), source_epoch=False): + assignment = _declared_alias_value(assignment, pymodule) + assign_node = assignment.type_hint or assignment.ast_node + if isinstance(assign_node, ast.pattern): + return pyobjects.get_unknown() + result = _follow_pyname( + assignment, pymodule, branch_context=branch_context, source_epoch=source_epoch + ) if result is None: return None pyname, pyobject = result @@ -169,17 +429,31 @@ def _follow_levels(assignment, pyobject): @_ignore_inferred -def _follow_pyname(assignment, pymodule, lineno=None): +def _follow_pyname( + assignment, pymodule, lineno=None, *, branch_context=(), source_epoch=False +): assign_node = assignment.type_hint or assignment.ast_node if lineno is None: lineno = _get_lineno_for_node(assign_node) holding_scope = pymodule.get_scope().get_inner_scope_for_line(lineno) + holding_kind = holding_scope.get_kind() + if branch_context or source_epoch: + context = pymodule._get_source_binding_index().context_for_node(assign_node) + if context is not None: + holding_scope = context.environment + holding_scope = evaluate.SourceValueScope( + holding_scope, + holding_scope, + {}, + branch_context=branch_context, + source_epoch=assign_node if source_epoch else None, + ) pyname = evaluate.eval_node(holding_scope, assign_node) if pyname is not None: result = pyname.get_object() if ( isinstance(result.get_type(), rope.base.builtins.Property) - and holding_scope.get_kind() == "Class" + and holding_kind == "Class" ): arg = pynames.UnboundName(pyobjects.PyObject(holding_scope.pyobject)) return pyname, result.get_type().get_property_object( diff --git a/rope/base/oi/transform.py b/rope/base/oi/transform.py index 5f9f310a0..3dcb68bab 100644 --- a/rope/base/oi/transform.py +++ b/rope/base/oi/transform.py @@ -4,7 +4,19 @@ import re import rope.base.builtins # Use full qualification for clarity. -from rope.base import exceptions +from rope.base import classfunctions, exceptions, pynames, pyobjects + + +def _is_instance_builtin(type_): + """Tell whether instances of a canonical builtin class use the generic form. + + Builtin classes with a dedicated textual form (list, dict, tuple, set, + str, file, iterators) keep it; every other builtin class, including the + canonical ``object`` and ``type`` models, is stored by its builtin name. + """ + return isinstance(type_, rope.base.builtins.BuiltinClass) and not hasattr( + PyObjectToTextual, type(type_).__name__ + "_to_textual" + ) class PyObjectToTextual: @@ -22,9 +34,19 @@ def transform(self, pyobject): """Transform a `PyObject` to textual form""" if pyobject is None: return ("none",) - object_type = type(pyobject) + if isinstance(pyobject, pyobjects.SampledValue): + # A read whose container position or cursor is unproved is not + # persisted; container element types below are (`_element`). + return ("unknown",) + # Subclasses without their own transformation, like literal + # values, are stored the way their nearest base class is. + for object_type in type(pyobject).__mro__: + method = getattr(self, object_type.__name__ + "_to_textual", None) + if method is not None: + break + else: + return ("unknown",) try: - method = getattr(self, object_type.__name__ + "_to_textual") return method(pyobject) except AttributeError: return ("unknown",) @@ -32,7 +54,18 @@ def transform(self, pyobject): def __call__(self, pyobject): return self.transform(pyobject) + def _element(self, pyobject): + return self.transform(pyobjects.unsampled(pyobject)) + def PyObject_to_textual(self, pyobject): + type_ = pyobject.get_type() + name = getattr(getattr(type_, "builtin", None), "__name__", None) + if ( + _is_instance_builtin(type_) + and name in rope.base.builtins.builtins + and rope.base.builtins.builtins[name].get_object() is type_ + ): + return ("builtin", "instance", name) if isinstance(pyobject.get_type(), rope.base.pyobjects.AbstractClass): result = self.transform(pyobject.get_type()) if result[0] == "defined": @@ -43,6 +76,13 @@ def PyObject_to_textual(self, pyobject): def PyFunction_to_textual(self, pyobject): return self._defined_to_textual(pyobject) + def BoundFunction_to_textual(self, pyobject): + return ( + "bound_function", + self.transform(pyobject.function), + self.transform(pyobject.receiver.get_object()), + ) + def PyClass_to_textual(self, pyobject): return self._defined_to_textual(pyobject) @@ -64,30 +104,28 @@ def PyPackage_to_textual(self, pyobject): return ("defined", self._get_pymodule_path(pyobject)) def List_to_textual(self, pyobject): - return ("builtin", "list", self.transform(pyobject.holding)) + return ("builtin", "list", self._element(pyobject.holding)) def Dict_to_textual(self, pyobject): return ( "builtin", "dict", - self.transform(pyobject.keys), - self.transform(pyobject.values), + self._element(pyobject.keys), + self._element(pyobject.values), ) def Tuple_to_textual(self, pyobject): - objects = [ - self.transform(holding) for holding in pyobject.get_holding_objects() - ] + objects = [self._element(holding) for holding in pyobject.get_holding_objects()] return tuple(["builtin", "tuple"] + objects) def Set_to_textual(self, pyobject): - return ("builtin", "set", self.transform(pyobject.holding)) + return ("builtin", "set", self._element(pyobject.holding)) def Iterator_to_textual(self, pyobject): - return ("builtin", "iter", self.transform(pyobject.holding)) + return ("builtin", "iter", self._element(pyobject.holding)) def Generator_to_textual(self, pyobject): - return ("builtin", "generator", self.transform(pyobject.holding)) + return ("builtin", "generator", self._element(pyobject.holding)) def Str_to_textual(self, pyobject): return ("builtin", "str") @@ -161,6 +199,12 @@ def builtin_generator_to_pyobject(self, textual): holding = self.transform(textual[2]) return rope.base.builtins.get_generator(holding) + def builtin_instance_to_pyobject(self, textual): + if textual[2] in rope.base.builtins.builtins: + type_ = rope.base.builtins.builtins[textual[2]].get_object() + if _is_instance_builtin(type_): + return rope.base.pyobjects.PyObject(type_) + def builtin_file_to_pyobject(self, textual): return rope.base.builtins.get_file() @@ -206,6 +250,12 @@ def instance_to_pyobject(self, textual): if type is not None: return rope.base.pyobjects.PyObject(type) + def bound_function_to_pyobject(self, textual): + function = self.transform(textual[1]) + receiver = self.transform(textual[2]) + if isinstance(function, pyobjects.PyFunction) and receiver is not None: + return classfunctions.BoundFunction(function, pynames.UnboundName(receiver)) + def _get_pymodule(self, path): resource = self.path_to_resource(path) if resource is not None: diff --git a/rope/base/oi/type_hinting/evaluate.py b/rope/base/oi/type_hinting/evaluate.py index e99310a37..3ceabb14c 100644 --- a/rope/base/oi/type_hinting/evaluate.py +++ b/rope/base/oi/type_hinting/evaluate.py @@ -4,6 +4,21 @@ from rope.base import utils as base_utils from rope.base.oi.type_hinting import utils +from rope.base.pyobjects import AbstractClass + + +class UnionType(AbstractClass): + """Attributes available from the alternatives in a union hint.""" + + def __init__(self, alternatives): + super().__init__() + self.alternatives = alternatives + + def get_attributes(self): + attributes = {} + for alternative in self.alternatives: + attributes.update(alternative.get_attributes()) + return attributes class SymbolBase: @@ -33,6 +48,7 @@ def __repr__(self): class SymbolTable: + @staticmethod def multi(func): def _inner(self, names, *a, **kw): for name in names.split(): @@ -119,8 +135,6 @@ def led(self, left, parser): self.first = left return self - multi = staticmethod(multi) # Just for code checker - symbol_table = SymbolTable() @@ -313,8 +327,12 @@ def evaluate(self, pyobject): @method(symbol("or")) # type: ignore @method(symbol("|")) def evaluate(self, pyobject): - # TODO: Implement me - raise NotImplementedError + alternatives = (self.first.evaluate(pyobject), self.second.evaluate(pyobject)) + if any(alternative is None for alternative in alternatives): + return None + return ( + utils.union_type(alternatives) if self.name == "|" else UnionType(alternatives) + ) class Compiler: diff --git a/rope/base/oi/type_hinting/factory.py b/rope/base/oi/type_hinting/factory.py index 85ff18e5b..6fa2e0637 100644 --- a/rope/base/oi/type_hinting/factory.py +++ b/rope/base/oi/type_hinting/factory.py @@ -1,6 +1,7 @@ from rope.base import utils from rope.base.oi.type_hinting import interfaces from rope.base.oi.type_hinting.providers import ( + annotations, composite, docstrings, inheritance, @@ -15,6 +16,7 @@ class TypeHintingFactory(interfaces.ITypeHintingFactory): @utils.saveit def make_param_provider(self): providers = [ + annotations.ParamProvider(self.make_resolver()), docstrings.ParamProvider( docstrings.DocstringParamParser(), self.make_resolver() ), @@ -27,6 +29,7 @@ def make_param_provider(self): @utils.saveit def make_return_provider(self): providers = [ + annotations.ReturnProvider(self.make_resolver()), docstrings.ReturnProvider( docstrings.DocstringReturnParser(), self.make_resolver() ), diff --git a/rope/base/oi/type_hinting/providers/annotations.py b/rope/base/oi/type_hinting/providers/annotations.py new file mode 100644 index 000000000..289210db2 --- /dev/null +++ b/rope/base/oi/type_hinting/providers/annotations.py @@ -0,0 +1,83 @@ +"""Resolve declared function annotations through the native hint resolver.""" + +from rope.base import ast, builtins, evaluate, pyobjects +from rope.base.oi.type_hinting import utils +from rope.base.oi.type_hinting.providers import interfaces + + +def _resolve_node(node, owner, resolver, late_owner): + if isinstance(node, ast.Constant): + if isinstance(node.value, str): + try: + expression = ast.parse(node.value, mode="eval").body + except SyntaxError: + return None + return _resolve_node(expression, late_owner, resolver, late_owner) + if node.value is None: + return builtins._runtime_class(type(None)) + return None + if isinstance(node, (ast.Name, ast.Attribute)): + return resolver(ast.unparse(node), owner) + if isinstance(node, ast.Subscript): + origin = _resolve_node(node.value, owner, resolver, late_owner) + if origin is None: + return None + arguments = ( + node.slice.elts if isinstance(node.slice, ast.Tuple) else [node.slice] + ) + if utils.is_annotated_type(origin): + if len(arguments) < 2: + return None + return _resolve_node(arguments[0], owner, resolver, late_owner) + if isinstance(origin, pyobjects.PyClass): + binding = evaluate.eval_node(utils.get_hint_scope(owner), node) + value = binding.get_object() if binding is not None else None + return value if isinstance(value, pyobjects.AbstractClass) else None + return utils.parametrize_type( + origin, + *( + _resolve_node(argument, owner, resolver, late_owner) + for argument in arguments + ), + ) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + alternatives = tuple( + _resolve_node(part, owner, resolver, late_owner) + for part in (node.left, node.right) + ) + return utils.union_type(alternatives) + return None + + +def _resolve_annotation(annotation, function, resolver): + if annotation is None: + return None + module = function.get_module() + context = module.get_source_binding_index().context_for_node(annotation) + owner = context.environment.pyobject if context is not None else function + try: + resolved = _resolve_node(annotation, owner, resolver, module) + except (TypeError, RecursionError): + resolved = None + if not isinstance(resolved, pyobjects.AbstractClass): + return pyobjects.get_unknown() + return resolved + + +class ParamProvider(interfaces.IParamProvider): + def __init__(self, resolver): + self._resolve = resolver + + def __call__(self, function, name): + args = function.get_ast().args + for argument in (*args.posonlyargs, *args.args, *args.kwonlyargs): + if argument.arg == name: + return _resolve_annotation(argument.annotation, function, self._resolve) + + +class ReturnProvider(interfaces.IReturnProvider): + def __init__(self, resolver): + self._resolve = resolver + + def __call__(self, function): + return _resolve_annotation(function.get_ast().returns, function, self._resolve) diff --git a/rope/base/oi/type_hinting/providers/pep0484_type_comments.py b/rope/base/oi/type_hinting/providers/pep0484_type_comments.py index d33566540..a89c72e8a 100644 --- a/rope/base/oi/type_hinting/providers/pep0484_type_comments.py +++ b/rope/base/oi/type_hinting/providers/pep0484_type_comments.py @@ -1,5 +1,6 @@ import re +import rope.base.pynamesdef from rope.base.oi.type_hinting import utils from rope.base.oi.type_hinting.providers import interfaces @@ -20,6 +21,8 @@ def __call__(self, pyname): """ from rope.base.oi.soi import _get_lineno_for_node + if not isinstance(pyname, rope.base.pynamesdef.AssignedName): + return None lineno = _get_lineno_for_node(pyname.assignments[0].ast_node) holding_scope = pyname.module.get_scope().get_inner_scope_for_line(lineno) line = holding_scope._get_global_scope()._scope_finder.lines.get_line(lineno) diff --git a/rope/base/oi/type_hinting/utils.py b/rope/base/oi/type_hinting/utils.py index 2381c8472..f15288868 100644 --- a/rope/base/oi/type_hinting/utils.py +++ b/rope/base/oi/type_hinting/utils.py @@ -1,25 +1,80 @@ from __future__ import annotations +import collections.abc import logging import sys +import types +import typing from typing import TYPE_CHECKING, Optional, Union import rope.base.utils as base_utils -from rope.base import evaluate +from rope.base import builtins, evaluate, pyobjects from rope.base.exceptions import AttributeNotFoundError -from rope.base.pyobjects import PyClass, PyDefinedObject, PyFunction, PyObject +from rope.base.pyobjects import ( + AbstractClass, + PyClass, + PyDefinedObject, + PyFunction, + PyObject, +) + + +def hint_instance(value): + """Materialize a class hint without wrapping a modeled instance twice.""" + if isinstance(value, AbstractClass) or value.get_type() is value: + return PyObject(value) + return value + + +def _hint_binding_value(binding): + """Authenticate stdlib declarations before assigning their native models.""" + value = binding.get_object() + if isinstance(value, pyobjects.SampledValue): + return None + module, line = binding.get_definition_location() + resource = module.get_resource() if module is not None else None + trusted = (typing, collections.abc, sys.modules.get("_collections_abc")) + if resource is not None and any( + base_utils.same_file_path(resource.real_path, item.__file__) + for item in trusted + if item is not None + ): + declared = module.get_scope().get_name("Sequence") + if declared.get_definition_location() == (module, line): + return builtins.get_sequence_type() + if resource is not None and base_utils.same_file_path( + resource.real_path, types.__file__ + ): + declared = module.get_scope().get_name("NoneType") + if declared.get_definition_location() == (module, line): + return builtins._runtime_class(type(None)) + if isinstance(value, (builtins.BuiltinClass, builtins.BuiltinUnknown)) and ( + value.builtin is collections.abc.Sequence or value.builtin is typing.Sequence + ): + return builtins.get_sequence_type() + return _hint_value(value) + + +def _hint_value(value): + if isinstance(value, builtins.GenericAlias): + origin = _hint_value(value.origin) + arguments = tuple(_hint_value(argument) for argument in value.arguments) + if origin is None or any( + not isinstance(argument, AbstractClass) for argument in arguments + ): + return None + return parametrize_type(origin, *arguments) + return value def get_super_func(pyfunc): if not isinstance(pyfunc.parent, PyClass): return - for cls in get_mro(pyfunc.parent)[1:]: - try: - superfunc = cls.get_attribute(pyfunc.get_name()).get_object() - except AttributeNotFoundError: - pass - else: + for cls in _source_ancestors(pyfunc.parent): + attributes = cls._get_own_attributes() + if pyfunc.get_name() in attributes: + superfunc = attributes[pyfunc.get_name()].get_object() if isinstance(superfunc, PyFunction): return superfunc @@ -34,9 +89,10 @@ def get_super_assignment(pyname): except TypeError: return else: - for super_pyclass in get_mro(pyclass)[1:]: - if attr_name in super_pyclass: - return super_pyclass[attr_name] + for super_pyclass in _source_ancestors(pyclass): + attributes = super_pyclass._get_own_attributes() + if attr_name in attributes: + return attributes[attr_name] def get_class_with_attr_name(pyname): @@ -45,6 +101,10 @@ def get_class_with_attr_name(pyname): :return: rope.base.pyobjectsdef.PyClass, str :rtype: tuple """ + import rope.base.pynamesdef + + if not isinstance(pyname, rope.base.pynamesdef.AssignedName): + return None lineno = get_lineno_for_node(pyname.assignments[0].ast_node) holding_scope = pyname.module.get_scope().get_inner_scope_for_line(lineno) pyobject = holding_scope.pyobject @@ -65,6 +125,19 @@ def get_lineno_for_node(assign_node): return 1 +def _source_ancestors(pyclass): + """Return the ancestors with source, in method resolution order + + A hierarchy Python refuses to create has no method resolution order; + its ancestors follow the bases as `get_mro()` lists them. + + """ + mro = pyclass.get_consistent_mro() + if mro is None: + mro = get_mro(pyclass) + return [cls for cls in mro[1:] if isinstance(cls, PyClass)] + + def get_mro(pyclass): # FIXME: to use real mro() result class_list = [pyclass] @@ -75,6 +148,55 @@ def get_mro(pyclass): return class_list +def _source_ancestors(pyclass): + """Return the ancestors with source, in method resolution order + + A hierarchy Python refuses to create has no method resolution order; + its ancestors follow the bases as `get_mro()` lists them. + + """ + from rope.base.pyobjectsdef import _inheritance_order + + return [cls for cls in _inheritance_order(pyclass)[1:] if isinstance(cls, PyClass)] + + +def get_hint_scope(pyobject): + from rope.base.sourcebindings import AnnotationEnvironment + + owner = pyobject.parent if isinstance(pyobject, PyFunction) else pyobject + scope = owner.get_scope() + if isinstance(scope, AnnotationEnvironment) and scope.policy.mode == "eager": + return evaluate.SourceValueScope(scope, scope, {}, source_epoch=scope.node) + if isinstance(scope, AnnotationEnvironment) and scope.policy.mode == "lazy": + # PEP 649 evaluates the annotation after its module has executed. + module = owner.get_module() + return evaluate.SourceValueScope( + scope, scope, {}, source_epoch=module.get_ast() + ) + return scope + + +def union_type(alternatives): + """Preserve PEP 604 only when native type operators govern both operands.""" + for alternative in alternatives: + if not isinstance(alternative, AbstractClass): + return None + metaclass = alternative.get_metaclass() + if not isinstance(metaclass, AbstractClass): + return None + for name in ("__or__", "__ror__"): + if name not in metaclass: + return None + method = metaclass[name].get_object() + if not isinstance( + method, builtins.BuiltinFunction + ) or method.builtin is not getattr(type, name): + return None + from rope.base.oi.type_hinting.evaluate import UnionType + + return UnionType(alternatives) + + def resolve_type( type_name: str, pyobject: Union[PyDefinedObject, PyObject], @@ -87,41 +209,41 @@ def resolve_type( else: deprecated_aliases = {"collections": "_collections_abc"} ret_type = None - logging.debug("Looking for %s", type_name) - if "." not in type_name: - try: - # XXX: this looks incorrect? It doesn't seem like it would work - # correctly if you have a type/class not defined in the - # module/global scope - ret_type = ( - pyobject.get_module().get_scope().get_name(type_name).get_object() - ) - except AttributeNotFoundError: - logging.exception("Cannot resolve type %s", type_name) - else: + logging.debug(f"Looking for {type_name}") + scope = get_hint_scope(pyobject) + root = type_name.partition(".")[0] + root_binding = scope.lookup(root) + if root_binding is not None: + binding = evaluate.eval_str(scope, type_name) + return _hint_binding_value(binding) if binding is not None else None + if "." in type_name: mod_name, attr_name = type_name.rsplit(".", 1) try: mod_finder = evaluate.ScopeNameFinder(pyobject.get_module()) mod = mod_finder._find_module(mod_name).get_object() - ret_type = mod.get_attribute(attr_name).get_object() + ret_type = _hint_binding_value(mod.get_attribute(attr_name)) except AttributeNotFoundError: if mod_name in deprecated_aliases: try: logging.debug( - "Looking for %s in %s", attr_name, deprecated_aliases[mod_name] + f"Looking for {attr_name} in {deprecated_aliases[mod_name]}" ) mod = mod_finder._find_module( deprecated_aliases[mod_name] ).get_object() ret_type = mod.get_attribute(attr_name).get_object() except AttributeNotFoundError: - logging.exception( - "Cannot resolve type %s in %s", attr_name, dir(mod) - ) + logging.exception(f"Cannot resolve type {attr_name} in {dir(mod)}") logging.debug("ret_type = %s", ret_type) return ret_type +def is_annotated_type(pyobject): + return builtins._is_runtime_class( + pyobject, typing.Annotated + ) or builtins._is_runtime_form(pyobject, typing.Annotated) + + class ParametrizeType: _supported_mapping = { "builtins.list": "rope.base.builtins.get_list", @@ -139,18 +261,36 @@ def __call__(self, pyobject, *args, **kwargs): :type pyobject: rope.base.pyobjects.PyObject :rtype: rope.base.pyobjects.PyDefinedObject | rope.base.pyobjects.PyObject or None """ + if is_annotated_type(pyobject): + return args[0] if len(args) >= 2 else None + if builtins._is_runtime_form(pyobject, typing.Optional): + if len(args) != 1 or args[0] is None: + return None + from rope.base.oi.type_hinting.evaluate import UnionType + + return UnionType((args[0], builtins._runtime_class(type(None)))) + if isinstance(pyobject, PyFunction): + return None type_factory = self._get_type_factory(pyobject) if type_factory: - parametrized_type = type_factory(*args, **kwargs) + parametrized_type = type_factory( + *( + hint_instance(argument) if argument is not None else None + for argument in args + ), + **kwargs, + ) if parametrized_type: - return parametrized_type - return pyobject + return parametrized_type.get_type() + return None def _get_type_factory(self, pyobject): - type_str = "{}.{}".format( - pyobject.get_module().get_name(), - pyobject.get_name(), - ) + if isinstance(pyobject, builtins.Sequence): + return builtins.get_sequence + module = pyobject.get_module() + if module is None: + return None + type_str = "{}.{}".format(module.get_name(), pyobject.get_name()) if type_str in self._supported_mapping: return base_utils.resolve(self._supported_mapping[type_str]) diff --git a/rope/base/project.py b/rope/base/project.py index c7f6a2f8e..83e49bf14 100644 --- a/rope/base/project.py +++ b/rope/base/project.py @@ -4,10 +4,12 @@ import json import os import sys +import sysconfig +import types import warnings from abc import ABC, abstractmethod from contextlib import ExitStack -from typing import TYPE_CHECKING, Optional, cast +from typing import TYPE_CHECKING, Optional import rope.base.fscommands # Use full qualification for clarity. import rope.base.resourceobserver as resourceobserver @@ -26,7 +28,7 @@ if TYPE_CHECKING: - from rope.base import pyobjectsdef + from rope.base import pyobjects, pyobjectsdef class _Project(ABC): @@ -34,9 +36,10 @@ class _Project(ABC): data_files: _DataFiles _custom_source_folders: list[Folder] - def __init__(self, fscommands): + def __init__(self, fscommands, external_project=None): self.observers = [] self.fscommands = fscommands + self.external_project = external_project self.prefs = Prefs() self.data_files = _DataFiles(self) self._custom_source_folders = [] @@ -53,28 +56,106 @@ def get_resource(self, resource_name: str) -> Resource: """ path = self._get_resource_path(resource_name) - if not os.path.exists(path): + if not self.fscommands.exists(path): raise exceptions.ResourceNotFoundError( "Resource <%s> does not exist" % resource_name ) - elif os.path.isfile(path): + elif self.fscommands.isfile(path): return File(self, resource_name) - elif os.path.isdir(path): + elif self.fscommands.isdir(path): return Folder(self, resource_name) else: raise exceptions.ResourceNotFoundError("Unknown resource " + resource_name) - def get_python_path_folders(self) -> list[Folder]: + def get_module( + self, name: str, folder: Optional[Folder] = None + ) -> pyobjects.AbstractModule: + """Returns a `PyObject` if the module was found.""" + # check if this is a builtin module + pymod = self.pycore.builtin_module(name) + if pymod is not None: + return pymod + module = self.find_module(name, folder) + if module is None: + observed = self._stdlib_module_alias(name, folder) + if observed is not None: + return observed + raise ModuleNotFoundError("Module %s not found" % name) + return self.pycore.resource_to_pyobject(module) + + def _stdlib_module_alias(self, name, folder): + """Read an existing stdlib module election without importing target code.""" + observed = sys.modules.get(name) + if not isinstance(observed, types.ModuleType) or observed.__spec__ is None: + return None + origin = observed.__spec__.name + if origin == name or sys.modules.get(origin) is not observed: + return None + if not hasattr(sys, "stdlib_module_names") or ( + origin.partition(".")[0] not in sys.stdlib_module_names + or name.partition(".")[0] not in sys.stdlib_module_names + ): + return None + parent_name, separator, _ = name.rpartition(".") + parent = sys.modules.get(parent_name) + filename = getattr(observed, "__file__", None) + parent_filename = getattr(parent, "__file__", None) + if ( + not separator + or not isinstance(filename, str) + or not isinstance(parent_filename, str) + ): + return None + stdlib = os.path.realpath(sysconfig.get_path("stdlib")) + filename = os.path.realpath(filename) + try: + contained = os.path.normcase( + os.path.commonpath((stdlib, filename)) + ) == os.path.normcase(stdlib) + except ValueError: + return None + if not contained: + return None + declared_parent = self.find_module(parent_name, folder) + declared_origin = self.find_module(origin, folder) + if ( + declared_parent is None + or declared_origin is None + or declared_origin.is_folder() + ): + return None + if declared_parent.is_folder(): + if not declared_parent.has_child("__init__.py"): + return None + declared_parent = declared_parent.get_child("__init__.py") + if not utils.same_file_path( + declared_parent.real_path, parent_filename + ) or not utils.same_file_path(declared_origin.real_path, filename): + return None + return self.pycore.resource_to_pyobject(declared_origin) + + def get_python_path_folders(self) -> list[Resource]: result = [] + external = ( + get_no_project() if self.external_project is None else self.external_project + ) for src in self.prefs.get("python_path", []) + sys.path: with contextlib.suppress(exceptions.ResourceNotFoundError): - src_folder = get_no_project().get_resource(src) + result.append(external.get_resource(src)) - # FIXME: it shouldn't be supported, but what if someone - # provided a .py file in python_path? - src_folder = cast(Folder, src_folder) + return result - result.append(src_folder) + # INFO: It was decided not to cache source folders, since: + # - Does not take much time when the root folder contains + # packages, that is most of the time + # - We need a separate resource observer; `self.observer` + # does not get notified about module and folder creations + def get_source_folders(self) -> list[Folder]: + """Returns project source folders""" + if self.root is None: + return [] + result = list(self._custom_source_folders) + result.extend(self.pycore._find_source_folders(self.root)) return result def validate(self, folder): @@ -135,6 +216,22 @@ def get_relative_module( raise ModuleNotFoundError("Module %s not found" % name) return self.pycore.resource_to_pyobject(module) + def find_module( + self, modname: str, folder: Optional[Folder] = None + ) -> Optional[Resource]: + """Returns a resource corresponding to the given module + + returns None if it can not be found + """ + + def folders(): + yield from self.get_source_folders() + yield from self.get_python_path_folders() + if folder is not None: + yield folder + + return _find_module_in_folders(folders(), modname) + def find_relative_module( self, modname: str, @@ -146,7 +243,7 @@ def find_relative_module( if modname == "": return folder else: - return _find_module_in_folder(folder, modname) + return _find_module_in_folders([folder], modname) def is_ignored(self, resource): return False @@ -189,7 +286,14 @@ class Project(_Project): file_list: _FileListCacher def __init__( - self, projectroot, fscommands=None, ropefolder=".ropeproject", **prefs + self, + projectroot, + fscommands=None, + ropefolder=".ropeproject", + external_project=None, + *, + config_files=True, + **prefs, ): """A rope project @@ -200,28 +304,61 @@ def __init__( - `ropefolder`: The name of the folder in which rope stores project configurations and data. Pass `None` for not using such a folder at all. + - `external_project`: Resource owner for Python-path modules. + Pass a separate ``NoProject`` with the same filesystem commands + to keep external and local reads in one captured filesystem. + - `config_files`: Pass ``False`` to take preferences only from + `prefs` over the defaults; no ``pyproject.toml``, ropefolder + ``config.py`` or global configuration is read. - `prefs`: Specify project preferences. These values overwrite config file preferences. """ if projectroot != "/": - projectroot = _realpath(projectroot).rstrip("/\\") + projectroot = ( + _realpath(projectroot) + if fscommands is None + else fscommands.realpath(projectroot) + ).rstrip("/\\") assert isinstance(projectroot, str) self._address = projectroot self._ropefolder_name = ropefolder - if not os.path.exists(self._address): - os.mkdir(self._address) - elif not os.path.isdir(self._address): + self._config_files = config_files + filesystem = ( + rope.base.fscommands.FileSystemCommands() + if fscommands is None + else fscommands + ) + if not filesystem.exists(self._address): + filesystem.create_folder(self._address) + elif not filesystem.isdir(self._address): raise exceptions.RopeError("Project root exists and" " is not a directory") if fscommands is None: fscommands = rope.base.fscommands.create_fscommands(self._address) - super().__init__(fscommands) + super().__init__(fscommands, external_project=external_project) self.ignored = _ResourceMatcher() self.file_list = _FileListCacher(self) + + extra_ignores = [ + p.pathlib.relative_to(self.root.pathlib) + for p in self.get_python_path_folders() + if p.pathlib.is_relative_to(self.root.pathlib) + and "site-packages" in p.pathlib.parts + ] + if extra_ignores: + # I'm using this approach, because self.prefs.add after _init_prefs + # does not work for some reason. + prefs.setdefault("ignored_resources", []).extend( + str(p) for p in extra_ignores + ) + self._init_prefs(prefs) if ropefolder is not None: self.prefs.add("ignored_resources", ropefolder) - self._init_source_folders() + + for path in self.prefs.get("source_folders", []): + folder = self.get_resource(path) + self._custom_source_folders.append(folder) def __repr__(self): return '<{}.{} "{}">'.format( @@ -230,63 +367,6 @@ def __repr__(self): self.address, ) - @utils.deprecated("Delete once deprecated functions are gone") - def _init_source_folders(self): - for path in self.prefs.get("source_folders", []): - folder = self.get_resource(path) - self._custom_source_folders.append(folder) - - # INFO: It was decided not to cache source folders, since: - # - Does not take much time when the root folder contains - # packages, that is most of the time - # - We need a separate resource observer; `self.observer` - # does not get notified about module and folder creations - def get_source_folders(self) -> list[Folder]: - """Returns project source folders""" - if self.root is None: - return [] - result = list(self._custom_source_folders) - result.extend(self.pycore._find_source_folders(self.root)) - return result - - def find_module( - self, - modname: str, - folder: Optional[Folder] = None, - ) -> Optional[Resource]: - """Returns a resource corresponding to the given module - - returns None if it can not be found - """ - for src in self.get_source_folders(): - module = _find_module_in_folder(src, modname) - if module is not None: - return module - for src in self.get_python_path_folders(): - module = _find_module_in_folder(src, modname) - if module is not None: - return module - if folder is not None: - module = _find_module_in_folder(folder, modname) - if module is not None: - return module - return None - - def get_module( - self, - name: str, - folder: Optional[Folder] = None, - ) -> pyobjectsdef.PyPackage | pyobjectsdef.PyModule: - """Returns a `PyObject` if the module was found.""" - # check if this is a builtin module - pymod = self.pycore.builtin_module(name) - if pymod is not None: - return pymod - module = self.find_module(name, folder) - if module is None: - raise ModuleNotFoundError("Module %s not found" % name) - return self.pycore.resource_to_pyobject(module) - def get_files(self) -> list[Resource]: return list(self.file_list.get_files()) @@ -299,7 +379,7 @@ def get_python_files(self) -> list[Resource]: ] def _get_resource_path(self, name: str) -> str: - return os.path.join(self._address, *name.split("/")) + return os.path.normpath(os.path.join(self._address, *name.split("/"))) def _init_ropefolder(self): if self.ropefolder is not None and not self.ropefolder.exists(): @@ -311,7 +391,11 @@ def _create_recursively(self, folder): folder.create() def _init_prefs(self, prefs): - config = get_config(self.root, self.ropefolder).parse() + config = ( + get_config(self.root, self.ropefolder).parse() + if self._config_files + else Prefs() + ) self.prefs = config self.prefs.add_callback("ignored_resources", self.ignored.set_patterns) self.ignored.set_patterns(self.prefs.ignored_resources) @@ -326,7 +410,7 @@ def _init_other_parts(self): # Forcing the creation of `self.pycore` to register observers self.pycore # pylint: disable=pointless-statement - def is_ignored(self, resource): + def is_ignored(self, resource: Resource): return self.ignored.does_match(resource) def sync(self): @@ -365,16 +449,17 @@ class NoProject(_Project): _no_project: Optional[NoProject] = None ropefolder: Optional[Folder] = None - def __init__(self): - fscommands = rope.base.fscommands.FileSystemCommands() + def __init__(self, fscommands=None): + if fscommands is None: + fscommands = rope.base.fscommands.FileSystemCommands() super().__init__(fscommands) def _get_resource_path(self, name: str) -> str: real_name = name.replace("/", os.path.sep) - return _realpath(real_name) + return self.fscommands.realpath(real_name) def get_resource(self, name: str) -> Resource: - universal_name = _realpath(name).replace(os.path.sep, "/") + universal_name = self.fscommands.realpath(name).replace(os.path.sep, "/") return super().get_resource(universal_name) def get_files(self) -> list[Resource]: @@ -489,23 +574,31 @@ def _realpath(path: str) -> str: return os.path.realpath(os.path.abspath(os.path.expanduser(path))) -def _find_module_in_folder(folder: Folder, modname: str) -> Optional[Resource]: - module = folder - packages = modname.split(".") - for pkg in packages[:-1]: - if module.is_folder() and module.has_child(pkg): - module = module.get_child(pkg) +def _find_module_in_folders(folders, modname): + for name in modname.split("."): + namespaces = [] + for folder in folders: + if not folder.is_folder(): + continue + package = folder.get_child(name) if folder.has_child(name) else None + if package is not None and package.is_folder(): + if ( + package.has_child("__init__.py") + and not package.get_child("__init__.py").is_folder() + ): + module = package + folders = [package] + break + namespaces.append(package) + if folder.has_child(name + ".py"): + source = folder.get_child(name + ".py") + if not source.is_folder(): + module = source + folders = [] + break else: - return None - if module.is_folder(): - if ( - module.has_child(packages[-1]) - and module.get_child(packages[-1]).is_folder() - ): - return module.get_child(packages[-1]) - elif ( - module.has_child(packages[-1] + ".py") - and not module.get_child(packages[-1] + ".py").is_folder() - ): - return module.get_child(packages[-1] + ".py") - return None + if not namespaces: + return None + module = namespaces[0] + folders = namespaces + return module diff --git a/rope/base/pycore.py b/rope/base/pycore.py index a6d6041cd..17fca8400 100644 --- a/rope/base/pycore.py +++ b/rope/base/pycore.py @@ -240,6 +240,7 @@ class _ModuleCache: def __init__(self, pycore): self.pycore = pycore self.module_map = {} + self.data_epoch = 0 self.pycore.cache_observers.append(self._invalidate_resource) self.observer = self.pycore.observer @@ -271,6 +272,7 @@ def get_pymodule(self, resource, force_errors=False): return result def forget_all_data(self): + self.data_epoch += 1 for pymodule in self.module_map.values(): pymodule._forget_concluded_data() diff --git a/rope/base/pynames.py b/rope/base/pynames.py index 3febd2197..65d5db1b5 100644 --- a/rope/base/pynames.py +++ b/rope/base/pynames.py @@ -56,7 +56,14 @@ class AssignmentValue: """An assigned expression""" def __init__( - self, ast_node, levels=None, evaluation="", assign_type=False, type_hint=None + self, + ast_node, + levels=None, + evaluation="", + assign_type=False, + type_hint=None, + *, + branch_context=None, ): """The `level` is `None` for simple assignments and is a list of numbers for tuple assignments for example in:: @@ -75,6 +82,7 @@ def __init__( self.evaluation = evaluation self.assign_type = assign_type self.type_hint = type_hint + self.branch_context = branch_context def get_lineno(self): return self.ast_node.lineno diff --git a/rope/base/pynamesdef.py b/rope/base/pynamesdef.py index 3aa8e7fc3..f3c32b7ff 100644 --- a/rope/base/pynamesdef.py +++ b/rope/base/pynamesdef.py @@ -14,17 +14,37 @@ def __init__(self, lineno=None, module=None, pyobject=None): self.lineno = lineno self.module = module self.assignments = [] + self.initial_binding = None + self._inference_assignment_limit = None + self._inference_assignment_context = () + self._inference_assignment_ambiguous = False self.pyobject = _Inferred( self._get_inferred, pynames._get_concluded_data(module) ) self.pyobject.set(pyobject) @utils.prevent_recursion(lambda: None) - def _get_inferred(self): + def _get_inferred(self, *, branch_context=(), read_node=None): if self.module is not None: - return rope.base.oi.soi.infer_assigned_object(self) - - def get_object(self): + return rope.base.oi.soi.infer_assigned_object( + self, branch_context=branch_context, read_node=read_node + ) + + def get_object(self, *, branch_context=(), read_node=None): + if self._inference_assignment_limit is not None: + value = rope.base.oi.soi.infer_assigned_object( + self, branch_context=branch_context, read_node=read_node + ) + return value if value is not None else rope.base.pyobjects.get_unknown() + if self.module is not None: + branch_context = rope.base.oi.soi.relevant_branch_context( + self, branch_context + ) + if branch_context or read_node is not None or self.initial_binding is not None: + value = self._get_inferred( + branch_context=branch_context, read_node=read_node + ) + return value if value is not None else rope.base.pyobjects.get_unknown() return self.pyobject.get() def get_definition_location(self): @@ -47,6 +67,7 @@ class ParameterName(pynames.ParameterName): def __init__(self, pyfunction, index): self.pyfunction = pyfunction self.index = index + self.assigned_value = None def get_object(self): result = self.pyfunction.get_parameter(self.index) diff --git a/rope/base/pyobjects.py b/rope/base/pyobjects.py index fbf1b34df..328ebe2da 100644 --- a/rope/base/pyobjects.py +++ b/rope/base/pyobjects.py @@ -19,6 +19,9 @@ def get_attribute(self, name): raise exceptions.AttributeNotFoundError("Attribute %s not found" % name) return self.get_attributes()[name] + def get_holding_objects(self): + return (self.type,) if self.type is not self else () + def get_module(self): return None @@ -120,6 +123,34 @@ def get_unknown(): return PyObject._unknown +class SampledValue(PyObject): + """A representative value whose container position or cursor is unproved.""" + + def __init__(self, representative, origin): + super().__init__(representative.get_type()) + self.representative = representative + self.origin = origin + + def get_attributes(self): + return self.representative.get_attributes() + + def get_module(self): + return self.representative.get_module() + + +def sampled_value(value, origin): + if value is None or value is get_unknown() or isinstance(value, SampledValue): + return value + return SampledValue(value, origin) + + +def unsampled(value): + """Return the representative behind a sampled read, else ``value``.""" + while isinstance(value, SampledValue): + value = value.representative + return value + + class AbstractClass(PyObject): def __init__(self): super().__init__(get_base_type("Type")) @@ -133,6 +164,204 @@ def get_doc(self): def get_superclasses(self): return [] + def get_metaclass(self): + """Return the governing class, or the unresolved metaclass witness.""" + from rope.base import builtins + + return builtins.builtins["type"].get_object() + + def get_bases(self): + """Return the resolved bases in declaration order + + Each item is either a class or an `UnresolvedBase` standing + for a base expression rope could not evaluate to a class. + Non-class bases with `__mro_entries__` expand into their returned + classes. PEP 695 classes also retain their implicit Generic base. + `get_superclasses()` returns only the classes. + + """ + return self.get_superclasses() + + def get_mro(self): + """Return the method resolution order of this class + + The order is the C3 linearization Python uses for ``__mro__``, + starting with this class. An `UnresolvedBase` keeps its place + in the order and contributes no ancestors. Builtin classes + contribute their runtime ``__mro__``; a class whose hierarchy + names neither ``object`` nor a builtin class does not end with + ``object``. + + Raises `rope.base.exceptions.InconsistentHierarchyError` when + Python would refuse to create the class. + + """ + return linearize(self, self.get_bases()) + + def get_consistent_mro(self): + """Return `get_mro()`, or `None` when Python would refuse the class + + Unlike `get_mro()`, this never raises + `rope.base.exceptions.InconsistentHierarchyError`. + + """ + return linearize(self, self.get_bases(), strict=False) + + def _get_own_attributes(self): + """Return the attributes this class defines itself + + These are the attributes the method resolution order selects + from this class. + + """ + return self.get_attributes() + + +class ConstructedClass(AbstractClass): + """A class produced by native class construction, without a source declaration.""" + + def __init__(self, name, bases, metaclass, attributes): + super().__init__() + self.name = name + self.bases = list(bases) + self.metaclass = metaclass + self.attributes = dict(attributes) + + def get_name(self): + return self.name + + def get_superclasses(self): + return self.bases + + def get_metaclass(self): + return self.metaclass + + def _get_own_attributes(self): + return self.attributes + + def get_attributes(self): + result = {} + mro = self.get_consistent_mro() + if mro is None: + for base in reversed(self.get_superclasses()): + result.update(base.get_attributes()) + result.update(self._get_own_attributes()) + return result + for ancestor in reversed(mro): + result.update(ancestor._get_own_attributes()) + return result + + +class UnresolvedBase(PyObject): + """A class base that rope could not evaluate to a class + + `node` is the base expression in the class statement. `pyobject` + is what that expression evaluated to, or `None` when it did not + evaluate at all. + + """ + + def __init__(self, node, pyobject=None): + super().__init__(get_base_type("Unknown")) + self.node = node + self.pyobject = pyobject + + def __eq__(self, obj): + # Transient by design (never cached in `_bases`/`_mro`), so each + # re-evaluation builds a fresh instance: unresolved bases identify + # by their source expression and evaluation, not by identity. + if isinstance(obj, UnresolvedBase): + return self.node is obj.node and self.pyobject == obj.pyobject + return NotImplemented + + def __ne__(self, obj): + result = self.__eq__(obj) + if result is NotImplemented: + return result + return not result + + def __hash__(self): + return hash((UnresolvedBase, id(self.node))) + + def get_mro(self): + return [self] + + def get_consistent_mro(self): + return self.get_mro() + + def _get_own_attributes(self): + return {} + + +def linearize(cls, bases, strict=True): + """Return the C3 linearization of `cls` declared with `bases` + + Raises `rope.base.exceptions.InconsistentHierarchyError` for repeated + bases and for bases whose orders cannot be merged. When `strict` is + false, these hierarchies, which Python refuses to create, return + `None` instead. + + """ + for index, base in enumerate(bases): + if any(base is previous for previous in bases[:index]): + if not strict: + return None + raise exceptions.InconsistentHierarchyError( + f"Duplicate base class in {_class_name(cls)}: {_class_name(base)}" + ) + if strict: + sequences = [list(base.get_mro()) for base in bases] + else: + sequences = [base.get_consistent_mro() for base in bases] + if any(sequence is None for sequence in sequences): + return None + sequences = [list(sequence) for sequence in sequences] + sequences.append(list(bases)) + result = [cls] + while True: + sequences = [sequence for sequence in sequences if sequence] + if not sequences: + return result + for sequence in sequences: + head = sequence[0] + if not any(head is item for other in sequences for item in other[1:]): + break + else: + if not strict: + return None + # Reporting the current heads identifies the pair of conflicting + # hierarchies; a bare message would hide which ones dispute the + # position. The sequence holding the unresolved head points to the + # transient base, and the modules tell where each sequence came from. + heads = [] + for sequence in sequences: + head = _class_name(sequence[0]) + if isinstance(sequence[0], UnresolvedBase): + head = f"{head} (node={type(sequence[0].node).__name__}:{getattr(sequence[0].node, 'lineno', '?')})" + origins = [] + for item in sequence[:3]: + try: + origins.append( + f"{_class_name(item)}@{item.get_module().get_name()}" + ) + except Exception: + origins.append(_class_name(item)) + heads.append(f"{head} | first3={origins}") + raise exceptions.InconsistentHierarchyError( + "Cannot create a consistent method resolution order " + f"for {_class_name(cls)} (blocked heads: {heads})" + ) + result.append(head) + for sequence in sequences: + if sequence[0] is head: + del sequence[0] + + +def _class_name(pyobject): + if isinstance(pyobject, UnresolvedBase): + return "" + return pyobject.get_name() + class AbstractFunction(PyObject): def __init__(self): @@ -147,6 +376,12 @@ def get_doc(self): def get_param_names(self, special_args=True): return [] + def get_positional_param_names(self): + return self.get_param_names(special_args=False) + + def get_keyword_param_names(self): + return self.get_param_names(special_args=False) + def get_returned_object(self, args): return get_unknown() @@ -194,6 +429,21 @@ def absolute_name(self): def _get_structural_attributes(self): if self.structural_attributes is None: self.structural_attributes = self._create_structural_attributes() + # A cache published before structural creation completes can only + # have been computed under the recursion guard (an empty temporary + # set or a hierarchy evaluated against a partial world), so it is + # stale. Invalidating it on completion gives later callers the real + # set, as the function attribute visitor already does. + if self.attributes.get() is not None: + self.attributes.set(None) + if self.concluded_attributes.get() is not None: + self.concluded_attributes.set(None) + bases = getattr(self, "_bases", None) + mro = getattr(self, "_mro", None) + if bases is not None and bases.get() is not None: + bases.set(None) + if mro is not None and mro.get() is not None: + mro.set(None) return self.structural_attributes @utils.prevent_recursion(lambda: {}) @@ -245,10 +495,18 @@ def get_doc(self) -> Optional[str]: def _get_defined_objects(self): if self.defineds is None: self._get_structural_attributes() - return self.defineds + # The recursion guard in _get_structural_attributes returns {} without + # completing creation, and objects without a visitor (for instance + # PyComprehension) also leave defineds as None. Callers iterate the + # result, so an empty mapping is the correct value on these paths. + return self.defineds if self.defineds is not None else {} def _create_structural_attributes(self): if self.visitor_class is None: + # Objects without a visitor (for instance PyComprehension) define + # nothing; otherwise defineds stays None and _get_defined_objects + # returns None, which breaks scope iteration. + self.defineds = {} return {} new_visitor = self.visitor_class(self.pycore, self) for child in ast.iter_child_nodes(self.ast_node): @@ -320,6 +578,14 @@ def _forget_concluded_data(self): for data in self.concluded_data: data._invalidate() + @utils.saveit + def _type_checking_tests(self): + """Branch tests of this module decided against `typing.TYPE_CHECKING` + + A decision reads other modules, so it is concluded data. + """ + return self._get_concluded_data() + def get_resource(self): return self.resource diff --git a/rope/base/pyobjectsdef.py b/rope/base/pyobjectsdef.py index 95bd15691..353588282 100644 --- a/rope/base/pyobjectsdef.py +++ b/rope/base/pyobjectsdef.py @@ -16,6 +16,104 @@ ) +def _inheritance_order(pyclass): + """Return the method resolution order of `pyclass`, or its bases walk + + A hierarchy Python refuses to create has no method resolution order; + its ancestors then follow the bases. + + """ + mro = pyclass.get_consistent_mro() + if mro is None: + from rope.base.oi.type_hinting.utils import get_mro + + mro = get_mro(pyclass) + return mro + + +def _mro_holds(mro, target): + """Tell whether ``mro`` holds the class ``target`` by runtime identity. + + A runtime class can be wrapped by distinct ``BuiltinClass`` objects + (``_runtime_class`` cache versus module attribute exposure); the + metaclass derivation compares the wrapped runtime classes, falling back + to object identity for source-defined classes. + """ + target_builtin = getattr(target, "builtin", None) + if target_builtin is not None: + for ancestor in mro: + if ancestor is target: + return True + if getattr(ancestor, "builtin", None) is target_builtin: + return True + return False + return target in mro + + +def _base_binding_at_position(pyname, base_node): + """Resolve a class base against the binding visible at its source position. + + Python evaluates base expressions at class-creation time. A module that + rebinds an imported name (``from x import c`` ... ``class F(c)`` ... + ``c = F``) resolves the base to the IMPORT binding: every recorded + assignment of the rebinding sits after the class statement. Names without + a retained pre-assignment binding, and names whose assignments genuinely + precede the base, keep the scope binding. + + """ + prior = getattr(pyname, "pre_assignment_binding", None) + if prior is None: + return pyname + for assignment in getattr(pyname, "assignments", ()) or (): + node = getattr(assignment, "type_hint", None) or assignment.ast_node + if (node.lineno, node.col_offset) <= (base_node.lineno, base_node.col_offset): + return pyname + return prior + + +def _scope_at_base_position(pycore, scope, base_node): + """Bind the root name of a compound base as it is at the class statement. + + ``s[T]`` or ``pkg.Base`` evaluates its root name before any position + correction can apply to the result; a later rebinding of that name (the + facade alias ``s = Derived``) would make the class its own base. + + """ + root = base_node + while isinstance(root, (ast.Subscript, ast.Attribute)): + root = root.value + if root is base_node or not isinstance(root, ast.Name): + return scope + pyname = scope.lookup(root.id) + if pyname is None: + return scope + positioned = _base_binding_at_position(pyname, base_node) + if positioned is pyname: + return scope + return rope.base.pyscopes.TemporaryScope(pycore, scope, {root.id: positioned}) + + +def _base_is_external(cls, base_node): + """Whether a failing base node lives in an installed third-party module. + + The base expression is evaluated in the declaring module's scope; when + that module's file sits outside the project root, the evaluation failure + is an analysis limitation over foreign code, never a defect of the + governed tree. + + """ + module = cls.get_module() + resource = module.get_resource() if module is not None else None + if resource is None: + return False + project = resource.project + if not hasattr(project, "address"): + # NoProject resources live outside any open project, such as + # installed libraries, so they are external by definition. + return True + return not str(resource.real_path).startswith(str(project.address)) + + class PyFunction(pyobjects.PyFunction): def __init__(self, pycore, ast_node, parent): rope.base.pyobjects.AbstractFunction.__init__(self) @@ -26,9 +124,10 @@ def __init__(self, pycore, ast_node, parent): ) self.returned = pynamesdef._Inferred(self._infer_returned) self.parameter_pynames = None + self.assigned_attributes = {} def _create_structural_attributes(self): - return {} + return self.assigned_attributes def _create_concluded_attributes(self): return {} @@ -45,11 +144,13 @@ def _infer_returned(self, args=None): return rope.base.oi.soi.infer_returned_object(self, args) def _handle_special_args(self, pyobjects): - if len(pyobjects) == len(self.arguments.args): - if self.arguments.vararg: - pyobjects.append(rope.base.builtins.get_list()) - if self.arguments.kwarg: - pyobjects.append(rope.base.builtins.get_dict()) + count = len(self.get_param_names(special_args=False)) + pyobjects[:] = pyobjects[:count] + pyobjects.extend([None] * (count - len(pyobjects))) + if self.arguments.vararg: + pyobjects.append(rope.base.builtins.get_list()) + if self.arguments.kwarg: + pyobjects.append(rope.base.builtins.get_dict()) def _set_parameter_pyobjects(self, pyobjects): if pyobjects is not None: @@ -76,13 +177,53 @@ def get_name(self): return self.get_ast().name def get_param_names(self, special_args=True): - # TODO: handle tuple parameters - result = [node.arg for node in self.arguments.args if isinstance(node, ast.arg)] - if special_args: - if self.arguments.vararg: - result.append(self.arguments.vararg.arg) - if self.arguments.kwarg: - result.append(self.arguments.kwarg.arg) + return [ + name + for kind, name in self.get_parameter_layout() + if special_args or kind not in ("vararg", "kwarg") + ] + + def get_parameter_layout(self): + """Return binding kinds in the order used by parameter object slots.""" + result = [("posonly", node.arg) for node in self.arguments.posonlyargs] + result.extend(("positional", node.arg) for node in self.arguments.args) + result.extend(("kwonly", node.arg) for node in self.arguments.kwonlyargs) + if self.arguments.vararg: + result.append(("vararg", self.arguments.vararg.arg)) + if self.arguments.kwarg: + result.append(("kwarg", self.arguments.kwarg.arg)) + return tuple(result) + + def get_positional_param_names(self): + return [ + name + for kind, name in self.get_parameter_layout() + if kind in ("posonly", "positional") + ] + + def get_keyword_param_names(self): + return [ + name + for kind, name in self.get_parameter_layout() + if kind in ("positional", "kwonly") + ] + + def get_parameter_defaults(self): + positional = self.arguments.posonlyargs + self.arguments.args + defaults = self.arguments.defaults + result = { + node.arg: default + for node, default in zip( + positional[len(positional) - len(defaults) :], defaults + ) + } + result.update( + (node.arg, default) + for node, default in zip( + self.arguments.kwonlyargs, self.arguments.kw_defaults + ) + if default is not None + ) return result def get_kind(self): @@ -96,10 +237,17 @@ def get_kind(self): if isinstance(self.parent, PyClass): for decorator in self.decorators: pyname = rope.base.evaluate.eval_node(scope, decorator) - if pyname == rope.base.builtins.builtins["staticmethod"]: + if pyname is None: + continue + decorator_object = pyname.get_object() + if not isinstance(decorator_object, rope.base.builtins.BuiltinClass): + continue + if decorator_object.builtin is staticmethod: return "staticmethod" - if pyname == rope.base.builtins.builtins["classmethod"]: + if decorator_object.builtin is classmethod: return "classmethod" + if self.get_name() == "__class_getitem__": + return "classmethod" return "method" return "function" @@ -132,32 +280,346 @@ def __init__(self, pycore, ast_node, parent): rope.base.pyobjects.AbstractClass.__init__(self) rope.base.pyobjects.PyDefinedObject.__init__(self, pycore, ast_node, parent) self.parent = parent - self._superclasses = self.get_module()._get_concluded_data() + self._bases = self.get_module()._get_concluded_data() + self._mro = self.get_module()._get_concluded_data() + self._metaclass = self.get_module()._get_concluded_data() + self._linearizing = False + self._receiver_attributes_collected = False + self._base_name_binding = None + + def _create_structural_attributes(self): + from rope.base import externaldata + + attributes = super()._create_structural_attributes() + attributes = externaldata.add_class_data(self, attributes) + if any( + rope.base.builtins._is_runtime_class(self, owner) + for owner in ( + rope.base.builtins.typing.Generic, + rope.base.builtins.typing.Protocol, + ) + ): + attributes["__class_getitem__"] = rope.base.builtins._class_getitem( + self, self.pycore.project, True + ) + return attributes + + def get_bases(self): + if self._bases.get() is None: + bases = self._get_bases() + if any( + isinstance(base, pyobjects.UnresolvedBase) and base.pyobject is None + for base in bases + ): + # A base that cannot be evaluated yet is a transient answer + # (inference is incomplete at that point); caching it would leave + # a permanent UnresolvedBase in the MRO. + return bases + self._bases.set(bases) + return self._bases.get() def get_superclasses(self): - if self._superclasses.get() is None: - self._superclasses.set(self._get_bases()) - return self._superclasses.get() + return [ + base + for base in self.get_bases() + if not isinstance(base, pyobjects.UnresolvedBase) + ] + + def get_mro(self): + """Return the native C3 order, retaining unresolved base witnesses. + + Linearize the current bases once and preserve hierarchy failures. + An order containing an unresolved base is partial and is never + cached as a complete method resolution order. + """ + return self._linearize(strict=True) + + def get_consistent_mro(self): + """Return `get_mro()`, or `None` when Python would refuse the class + + Cyclic, repeated and unmergeable bases give `None` instead of + raising `rope.base.exceptions.InconsistentHierarchyError`. + """ + return self._linearize(strict=False) + + def _linearize(self, strict): + if self._mro.get() is None: + # Evaluating the bases may look up attributes of this class; + # only the linearization of the ancestors can reach it again. + bases = self.get_bases() + if self._linearizing: + if not strict: + return None + raise exceptions.InconsistentHierarchyError( + "Cyclic class inheritance: %s" % self.get_name() + ) + self._linearizing = True + try: + mro = pyobjects.linearize(self, bases, strict=strict) + finally: + self._linearizing = False + if mro is None or any( + isinstance(base, pyobjects.UnresolvedBase) for base in mro + ): + return mro + self._mro.set(mro) + return self._mro.get() + + def invalidate_bases(self): + """Drop the cached bases and MRO so the next read recomputes them. + + Call this after the inputs defining the hierarchy change. + """ + self._bases.set(None) + self._mro.set(None) + + def get_mro(self): + return self._linearize(strict=True) + + def get_consistent_mro(self): + return self._linearize(strict=False) + + def _linearize(self, strict): + if self._mro.get() is None: + bases = self.get_superclasses() + if self._linearizing: + if not strict: + return None + raise exceptions.InconsistentHierarchyError( + "Cyclic class inheritance: %s" % self.get_name() + ) + self._linearizing = True + try: + mro = pyobjects.linearize(self, bases, strict=strict) + finally: + self._linearizing = False + if mro is None: + return None + self._mro.set(mro) + return self._mro.get() def get_name(self): return self.get_ast().name + def get_metaclass(self): + if self._metaclass.get() is None: + winner = super().get_metaclass() + for keyword in self.ast_node.keywords: + if keyword.arg == "metaclass": + binding = rope.base.evaluate.eval_node( + self.parent.get_scope(), keyword.value + ) + if binding is None or not isinstance( + binding.get_object(), pyobjects.AbstractClass + ): + self._metaclass.set( + pyobjects.UnresolvedBase( + keyword.value, + binding.get_object() if binding is not None else None, + ) + ) + return self._metaclass.get() + winner = binding.get_object() + for base in self.get_bases(): + if isinstance(base, pyobjects.UnresolvedBase): + self._metaclass.set(base) + return base + candidate = base.get_metaclass() + if not isinstance(candidate, pyobjects.AbstractClass): + self._metaclass.set(candidate) + return candidate + if _mro_holds(candidate.get_mro(), winner): + winner = candidate + elif not _mro_holds(winner.get_mro(), candidate): + raise exceptions.InconsistentHierarchyError( + "Conflicting metaclasses: %s" % self.get_name() + ) + self._metaclass.set(winner) + return self._metaclass.get() + + def _get_structural_attributes(self): + attributes = super()._get_structural_attributes() + if ( + self.structural_attributes is not None + and not self._receiver_attributes_collected + ): + # Resolve decorators only after all class bindings exist and the + # structural visitor's recursion guard has been released. + self._receiver_attributes_collected = True + self.attributes.set(None) + visitor = self.visitor_class(self.pycore, self) + visitor.names = attributes + try: + for defined in self.defineds: + if ( + isinstance(defined, PyFunction) + and defined.get_kind() != "staticmethod" + ): + visitor.visit_method_attributes(defined.get_ast()) + finally: + # get_kind() can cache a view before the receiver fields exist. + self.attributes.set(None) + return attributes + def _create_concluded_attributes(self): result = {} - for base in reversed(self.get_superclasses()): - result.update(base.get_attributes()) + mro = self.get_consistent_mro() + if mro is None: + # A hierarchy Python refuses to create has no method resolution + # order; its attributes are inherited along the bases, as rope + # always did for such classes. + for base in reversed(self.get_superclasses()): + result.update(base.get_attributes()) + return result + for ancestor in reversed(mro[1:]): + result.update(ancestor._get_own_attributes()) return result + def _get_own_attributes(self): + return self._get_structural_attributes() + def _get_bases(self): + if rope.base.builtins._is_runtime_class( + self, rope.base.builtins.typing.Generic + ): + return rope.base.builtins._runtime_class( + rope.base.builtins.typing.Generic + ).get_superclasses() + bindings = [] + for base_node in self.ast_node.bases: + scope = self.parent.get_scope() + if self._base_name_binding is not None: + from rope.base.sourcebindings import TypeParameterEnvironment + + context = ( + self.get_module() + ._get_source_binding_index() + .context_for_node(base_node) + ) + if context is not None: + scope = context.environment + names = {self.get_name(): self._base_name_binding} + if isinstance(scope, TypeParameterEnvironment): + names.update(scope.get_defined_names()) + scope = rope.base.pyscopes.TemporaryScope(self.pycore, scope, names) + scope = _scope_at_base_position(self.pycore, scope, base_node) + try: + evaluated = rope.base.evaluate.eval_node(scope, base_node) + bindings.append(_base_binding_at_position(evaluated, base_node)) + except Exception as error: + try: + module = self.get_module() + expression = ast.get_source_segment(module.source_code, base_node) + note = ( + f"while evaluating base {expression!r} of " + f"{module.get_name()}.{self.get_name()} " + f"at line {base_node.lineno}" + ) + except Exception as diagnostic_error: + note = f"base evaluation context failed: {diagnostic_error!r}" + error.__dict__.setdefault("__notes__", []).append(note) + # A base outside the project that analysis cannot evaluate (for + # instance a generic alias from an installed library) becomes an + # UnresolvedBase, which callers already handle, instead of failing + # the whole indexing. Classes of the project still propagate the + # error, since it is a real defect there. + if _base_is_external(self, base_node): + bindings.append(None) + continue + raise + original = rope.base.builtins.get_tuple( + *( + binding.get_object() if binding is not None else None + for binding in bindings + ) + ) + generic_aliases = [ + value + for value in original.get_type().get_holding_objects() + if isinstance(value, rope.base.builtins.GenericAlias) + and rope.base.builtins._is_runtime_class( + value.origin, rope.base.builtins.typing.Generic + ) + ] + if generic_aliases and any( + alias != generic_aliases[0] for alias in generic_aliases[1:] + ): + raise TypeError("Cannot inherit from Generic[...] multiple times") result = [] - for base_name in self.ast_node.bases: - base = rope.base.evaluate.eval_node(self.parent.get_scope(), base_name) - if ( - base is not None - and base.get_object().get_type() - == rope.base.pyobjects.get_base_type("Type") + for base_node, binding in zip(self.ast_node.bases, bindings): + if binding is None: + result.append(pyobjects.UnresolvedBase(base_node)) + else: + value = binding.get_object() + if isinstance(value, pyobjects.AbstractClass): + result.append(value) + elif "__mro_entries__" in value: + method, receiver = rope.base.evaluate.get_callable( + value["__mro_entries__"] + ) + if method is None: + raise TypeError("__mro_entries__ must be callable") + if receiver is None: + receiver = pynamesdef.UnboundName(value) + args = arguments.ObjectArguments([pynamesdef.UnboundName(original)]) + args = arguments.bind_method( + receiver, + method, + args, + self.parent.get_scope(), + owner=receiver.get_object().get_type(), + ) + replacement = method.get_returned_object(args) + if replacement is None: + result.append(pyobjects.UnresolvedBase(base_node, value)) + continue + sequence = replacement.get_type() + if not isinstance(sequence, rope.base.builtins.Tuple): + raise TypeError("__mro_entries__ must return a tuple") + entries = sequence.get_holding_objects() + # ``__mro_entries__`` derivations may hand back the raw + # typing origin (a GenericAlias head or a module-level + # alias object) instead of a rope class. A non-class + # entry occupies its MRO slot as an unresolved witness — + # raising here would abort the whole hierarchy walk for + # a base Python itself accepts at runtime. + classes = [ + entry + for entry in entries + if isinstance(entry, pyobjects.AbstractClass) + ] + result.extend(classes) + if len(classes) != len(entries): + result.append(pyobjects.UnresolvedBase(base_node, value)) + else: + if isinstance(base_node, ast.Subscript): + origin_binding = rope.base.evaluate.eval_node( + scope, base_node.value + ) + origin = ( + origin_binding.get_object() + if origin_binding is not None + else None + ) + if isinstance(origin, pyobjects.AbstractClass): + # PEP 560: a parameterized base contributes the ancestry of its + # origin class. When the owner's __class_getitem__ is opaque (for + # instance pydantic builds dynamic submodels), the value is neither + # a class nor exposes __mro_entries__, and the origin is the correct + # lower bound for the MRO. + result.append(origin) + continue + result.append(pyobjects.UnresolvedBase(base_node, value)) + if getattr(self.ast_node, "type_params", ()): + generic = self.pycore.project.get_module("typing")["Generic"].get_object() + if not isinstance(generic, pyobjects.AbstractClass): + result.append(pyobjects.UnresolvedBase(self.ast_node, generic)) + elif not any( + isinstance(base, pyobjects.AbstractClass) + and generic in _inheritance_order(base) + for base in result ): - result.append(base.get_object()) + result.append(generic) return result def _create_scope(self): @@ -214,6 +676,36 @@ def _create_concluded_attributes(self): def _create_scope(self): return rope.base.pyscopes.GlobalScope(self.pycore, self) + def get_source_binding_index(self): + return self._get_source_binding_index()._complete() + + @utils.saveit + def _get_source_binding_index(self): + from rope.base.sourcebindings import SourceBindingIndex + + return SourceBindingIndex( + self, _defer_bodies=not self.requires_header_bindings() + ) + + @utils.saveit + def requires_header_bindings(self): + """Whether walrus header names need binding-index preparation. + + The raw ``":=" in source_code`` scan answers a per-module constant, + so attribute and scope reads must not repeat it per call. + """ + return ":=" in self.source_code + + def get_attributes(self): + if self.requires_header_bindings(): + self._get_source_binding_index().prepare_header_bindings() + return super().get_attributes() + + def get_attribute(self, name): + if self.requires_header_bindings(): + self._get_source_binding_index().prepare_header_bindings() + return super().get_attribute(name) + @property @utils.saveit def lines(self): @@ -235,15 +727,27 @@ def __init__(self, pycore, resource=None, force_errors=False): self.resource = resource init_dot_py = self._get_init_dot_py() if init_dot_py is not None: + self.source_code = pycore.project.get_pymodule( + init_dot_py, force_errors=force_errors + ).source_code ast_node = pycore.project.get_pymodule( init_dot_py, force_errors=force_errors ).get_ast() else: - ast_node = ast.parse("\n") + # A namespace package has no source of its own; model it as + # an empty module so scope and line queries work on it. + self.source_code = "\n" + ast_node = ast.parse(self.source_code) + self.star_imports = [] + self.visitor_class = _GlobalVisitor + self.coding = fscommands.read_str_coding(self.source_code) super().__init__(pycore, ast_node, resource) def _create_structural_attributes(self): result = {} + # A package defines no module-level objects of its own; its names + # come from extension/builtin submodules and its folder children. + self.defineds = [] modname = rope.base.libutils.modname(self.resource) extension_submodules = self.pycore._builtin_submodules(modname) for name, module in extension_submodules.items(): @@ -278,8 +782,26 @@ def _get_init_dot_py(self): else: return None + @property + @utils.saveit + def lines(self): + """A `SourceLinesAdapter` over the package source""" + return rope.base.codeanalyze.SourceLinesAdapter(self.source_code) + + @property + @utils.saveit + def logical_lines(self): + """A `LogicalLinesFinder` over the package source""" + return rope.base.codeanalyze.CachingLogicalLineFinder(self.lines) + def _create_scope(self): - return self.get_module().get_scope() + module = self.get_module() + if module is self: + # A namespace package has no __init__.py to delegate to; + # fall back to its own (empty) module scope instead of + # recursing through get_scope() forever. + return rope.base.pyscopes.GlobalScope(self.pycore, self) + return module.get_scope() def get_module(self): init_dot_py = self._get_init_dot_py() @@ -364,13 +886,21 @@ class _AssignVisitor(ast.RopeNodeVisitor): def __init__(self, scope_visitor): self.scope_visitor = scope_visitor self.assigned_ast = None + self.assigned_targets = () def _Assign(self, node): self.assigned_ast = node.value + self.assigned_targets = node.targets for child_node in node.targets: self.visit(child_node) _ExpressionVisitor(self.scope_visitor).visit(node.value) + def _TypeAlias(self, node): + # the alias value is not recorded as an assigned value: a type + # alias is a distinct TypeAliasType object whose value is evaluated + # lazily, so substituting the value for the alias name is unsafe + self.visit(node.name) + def _assigned(self, name, assignment=None): self.scope_visitor._assigned(name, assignment) @@ -389,7 +919,61 @@ def _Tuple(self, node): self._assigned(name, assignment) def _Attribute(self, node): - pass + visitor = self.scope_visitor + if not isinstance(visitor, _GlobalVisitor) or not isinstance( + node.value, ast.Name + ): + return + receiver = visitor.names.get(node.value.id) + if not isinstance(receiver, pynamesdef.DefinedName): + return + function = receiver.get_object() + if ( + not isinstance(function, PyFunction) + or function.parent is not visitor.owner_object + ): + return + if visitor.depth != 1 or len(self.assigned_targets) != 1: + function.assigned_attributes[node.attr] = self._unresolved_attribute( + function, node, "conditional or chained assignment" + ) + function.attributes.set(None) + return + builtins_scope = rope.base.pyscopes.Scope( + visitor.pycore, rope.base.builtins.builtins, None + ) + scope = rope.base.pyscopes.TemporaryScope( + visitor.pycore, builtins_scope, visitor.names + ) + scope.pyobject = visitor.owner_object + value = rope.base.evaluate.eval_node(scope, self.assigned_ast) + if value is None: + function.assigned_attributes[node.attr] = self._unresolved_attribute( + function, node, "unresolved assignment value" + ) + function.attributes.set(None) + return + from rope.base.sourcebindings import SourceValueName + + function.assigned_attributes[node.attr] = SourceValueName( + value, value.get_object() + ) + function.attributes.set(None) + + @staticmethod + def _unresolved_attribute(function, target, reason): + module = function.get_module() + line = target.lineno + message = ( + "Unresolved function attribute write: " + f"{module.get_name()}:{line}: " + f"{function.get_name()}.{target.attr} ({reason})" + ) + + def unresolved(): + raise exceptions.RopeError(message) + + return pynamesdef.EvaluatedName(unresolved, module=module, lineno=line) def _Subscript(self, node): pass @@ -414,6 +998,7 @@ def get_module(self): def _ClassDef(self, node): pyclass = PyClass(self.pycore, node, self.owner_object) + pyclass._base_name_binding = self.names.get(node.name) self.names[node.name] = pynamesdef.DefinedName(pyclass) self.defineds.append(pyclass) @@ -451,9 +1036,34 @@ def _Assign(self, node): def _AnnAssign(self, node): _AnnAssignVisitor(self).visit(node) + def _TypeAlias(self, node): + _AssignVisitor(self).visit(node) + pyname = self.names[node.name.id] + if ( + isinstance(pyname, pynamesdef.AssignedName) + and pyname.get_definition_location()[1] is None + ): + pyname.lineno = node.lineno + def _AugAssign(self, node): pass + def _add_match_capture(self, node, name): + if name is not None: + assignment = pynamesdef.AssignmentValue(node) + self._assigned(name, assignment) + + def _MatchAs(self, node): + self._add_match_capture(node, node.name) + self.generic_visit(node) + + def _MatchStar(self, node): + self._add_match_capture(node, node.name) + + def _MatchMapping(self, node): + self._add_match_capture(node, node.rest) + self.generic_visit(node) + def _For(self, node): self._update_evaluated(node.target, node.iter, ".__iter__().next()") for child in node.body + node.orelse: @@ -464,10 +1074,25 @@ def _AsyncFor(self, node): def _assigned(self, name, assignment): pyname = self.names.get(name, None) - if pyname is None: - pyname = pynamesdef.AssignedName(module=self.get_module()) + if pyname is None or not isinstance(pyname, pynamesdef.AssignedName): + # Python rebinds the name on assignment: an earlier import + # binding (ImportedName/ImportedModule) must not shadow the + # module's own later assignment (runtime semantics). + replacement = pynamesdef.AssignedName(module=self.get_module()) + # The replaced binding stays visible for source positions BEFORE + # the first assignment (class bases, mid-module lookups): class + # creation evaluates the import, not the later rebind. + replacement.pre_assignment_binding = pyname + pyname = replacement if isinstance(pyname, pynamesdef.AssignedName): if assignment is not None: + node = assignment.type_hint or assignment.ast_node + context = ( + self.get_module()._get_source_binding_index().context_for_node(node) + ) + assignment.branch_context = ( + context.branch_context if context is not None else None + ) pyname.assignments.append(assignment) self.names[name] = pyname @@ -583,14 +1208,40 @@ def _get_pyobject(self, node): class _GlobalVisitor(_ScopeVisitor): - pass + """Build module bindings without treating inactive version imports as live. + + A direct import-only guard may select a branch only after an import-only + standard-library prelude authenticates its ``sys`` binding and runtime. + The source AST remains complete; mixed scopes, TYPE_CHECKING and unproved + conditions retain their existing traversal and declaration identities. + """ + + def __init__(self, pycore, owner_object): + super().__init__(pycore, owner_object) + self.depth = 0 + + def _If(self, node): + selected = rope.base.evaluate._header_version_guard( + self.get_module(), self.names, node + ) + if selected is None: + return self.generic_visit(node) + for statement in node.body if selected else node.orelse: + self.visit(statement) + + def visit(self, node): + self.depth += 1 + try: + return super().visit(node) + finally: + self.depth -= 1 class _ClassVisitor(_ScopeVisitor): - def _FunctionDef(self, node): - _ScopeVisitor._FunctionDef(self, node) - if len(node.args.args) > 0: - first = node.args.args[0] + def visit_method_attributes(self, node): + positional = node.args.posonlyargs + node.args.args + if positional: + first = positional[0] new_visitor = None if isinstance(first, ast.arg): new_visitor = _ClassInitVisitor(self, first.arg) diff --git a/rope/base/pyscopes.py b/rope/base/pyscopes.py index 565541903..bb4f653e5 100644 --- a/rope/base/pyscopes.py +++ b/rope/base/pyscopes.py @@ -1,6 +1,4 @@ -import rope.base.builtins # Use full qualification for clarity. from rope.base import ast, codeanalyze, exceptions, pynames, utils -from rope.refactor import patchedast class Scope: @@ -31,13 +29,22 @@ def __contains__(self, key): """The same as ``key in self.get_names()``""" return key in self.get_names() - @utils.saveit def get_scopes(self): """Return the subscopes of this scope The returned scopes should be sorted by the order they appear. """ - return self._create_scopes() + scopes = getattr(self, "_scopes", None) + if scopes is None: + scopes = self._create_scopes() + # While the owner is being created, the recursion guard in + # _get_structural_attributes returns a temporary empty set; caching + # that answer (saveit) would leave a permanent empty list for + # structural references. The cache is only stored once the owner's + # defined objects exist. + if getattr(self.pyobject, "defineds", None) is not None: + self._scopes = scopes + return scopes def lookup(self, name): if name in self.get_names(): @@ -46,6 +53,10 @@ def lookup(self, name): return self.parent._propagated_lookup(name) return None + def get_binding_scopes(self): + module = self._get_global_scope().pyobject + return module.get_source_binding_index().children_of(self) + def get_propagated_names(self): """Return the visible names of this scope @@ -83,6 +94,7 @@ def get_body_start(self): return body[0].lineno return self.get_start() + @utils.saveit def get_end(self): pymodule = self._get_global_scope().pyobject return pymodule.logical_lines.logical_line_in(self.logical_end)[1] @@ -99,14 +111,24 @@ def get_logical_end(self): def get_kind(self): pass + @utils.saveit def get_region(self): - self._calculate_scope_regions_for_module() + global_scope = self._get_global_scope() node = self.pyobject.get_ast() - region = patchedast.node_region(node) - return region - - def _calculate_scope_regions_for_module(self): - self._get_global_scope()._calculate_scope_regions() + if isinstance(node, ast.Module): + return (0, len(global_scope._get_source())) + start, end = global_scope._ast_lines[node] + if start is None or end is None: + raise exceptions.RopeError("Scope AST has no complete source positions") + decorators = getattr(node, "decorator_list", ()) + if decorators: + decorator_start, _ = global_scope._ast_lines[decorators[0]] + if decorator_start is None: + raise exceptions.RopeError("Decorator AST has no source positions") + start = global_scope._region_source.rfind_token("@", 0, decorator_start) + if start is None: + raise exceptions.RopeError("Scope decorator has no source marker") + return (start, end) def in_region(self, offset): """Checks if offset is in scope region""" @@ -119,6 +141,7 @@ class GlobalScope(Scope): def __init__(self, pycore, module): super().__init__(pycore, module, None) self.names = module._get_concluded_data() + self._inner_scopes_for_lines = {} def get_start(self): return 1 @@ -134,10 +157,21 @@ def get_name(self, name): return self.builtin_names[name] raise exceptions.NameNotFoundError("name %s not found" % name) + @property @utils.saveit - def _calculate_scope_regions(self): - source = self._get_source() - patchedast.patch_ast(self.pyobject.get_ast(), source) + def _ast_lines(self): + return codeanalyze.ASTLinesAdapter(self._get_source()) + + @property + @utils.saveit + def _region_source(self): + # Deferred import: rope.refactor's package init pulls the analysis + # stack, which reaches evaluate; evaluate subclasses this module's + # TemporaryScope at import time, so a module-level import here is + # circular when pyscopes loads first. + from rope.refactor import patchedast + + return patchedast._Source(self._get_source()) def _get_source(self): return self.pyobject.source_code @@ -150,11 +184,23 @@ def get_names(self): return self.names.get() def get_inner_scope_for_line(self, lineno, indents=None): - return self._scope_finder.get_holding_scope(self, lineno, indents) + # The holding scope for a (line, indents) pair is a structural + # property of the module: inference follows thousands of name paths + # through the same lines, so the tree walk runs once per pair for the + # lifetime of this module scope (rebuilt with its pymodule). + key = (lineno, indents) + cached = self._inner_scopes_for_lines.get(key) + if cached is None: + cached = self._scope_finder.get_holding_scope(self, lineno, indents) + self._inner_scopes_for_lines[key] = cached + return cached def get_inner_scope_for_offset(self, offset): return self._scope_finder.get_holding_scope_for_offset(self, offset) + def get_structural_scope_for_offset(self, offset): + return self._scope_finder.get_holding_scope_for_offset(self, offset) + @property @utils.saveit def _scope_finder(self): @@ -162,9 +208,32 @@ def _scope_finder(self): @property def builtin_names(self): + # Deferred import: builtins imports evaluate, whose SourceValueScope + # subclasses this module's TemporaryScope at import time; a module-level + # import here re-enters a half-initialized pyscopes (circular). + import rope.base.builtins # Use full qualification for clarity. + return rope.base.builtins.builtins.get_attributes() +class BuiltinModuleScope(Scope): + """Module scope over a C/builtin module. + + Builtin modules carry no source, concluded data, or child scopes; their + names are the live attribute mapping of the native module. + """ + + def __init__(self, pyobject): + super().__init__(getattr(pyobject, "pycore", None), pyobject, None) + self.names = pyobject.attributes + + def get_start(self): + return 1 + + def get_kind(self): + return "Module" + + class ComprehensionScope(Scope): def __init__(self, pycore, pyobject, visitor): super().__init__(pycore, pyobject, pyobject.parent.get_scope()) @@ -211,15 +280,24 @@ def __init__(self, pycore, pyobject, visitor): def _get_names(self): if self.names is None: self._visit_function() + module = self.pyobject.get_module() + if module.requires_header_bindings(): + module._get_source_binding_index().prepare_header_bindings(self) return self.names def _visit_function(self): if self.names is None: new_visitor = self.visitor(self.pycore, self.pyobject) - for n in ast.iter_child_nodes(self.pyobject.get_ast()): + for n in self.pyobject.get_ast().body: new_visitor.visit(n) self.names = new_visitor.names - self.names.update(self.pyobject.get_parameters()) + parameters = self.pyobject.get_parameters() + for name, parameter in parameters.items(): + assigned = self.names.get(name) + if isinstance(assigned, pynames.AssignedName): + assigned.initial_binding = parameter + parameter.assigned_value = assigned + self.names.update(parameters) self.returned_asts = new_visitor.returned_asts self.is_generator = new_visitor.generator self.defineds = new_visitor.defineds diff --git a/rope/base/resourceobserver.py b/rope/base/resourceobserver.py index 8b2186d2a..8ef980301 100644 --- a/rope/base/resourceobserver.py +++ b/rope/base/resourceobserver.py @@ -250,14 +250,16 @@ class ChangeIndicator: def get_indicator(self, resource): """Return the modification time and size of a `Resource`.""" path = resource.real_path + filesystem = resource.project.fscommands + state = filesystem.stat(path) # on dos, mtime does not change for a folder when files are added - if os.name != "posix" and os.path.isdir(path): + if os.name != "posix" and filesystem.isdir(path): return ( - os.path.getmtime(path), - len(os.listdir(path)), - os.path.getsize(path), + state.st_mtime, + len(filesystem.listdir(path)), + state.st_size, ) - return (os.path.getmtime(path), os.path.getsize(path)) + return (state.st_mtime, state.st_size) class _Changes: diff --git a/rope/base/resources.py b/rope/base/resources.py index 1acbf1730..94668a362 100644 --- a/rope/base/resources.py +++ b/rope/base/resources.py @@ -75,7 +75,7 @@ def create(self): """Create this resource""" def exists(self): - return os.path.exists(self) + return self.project.fscommands.exists(self.real_path) @property def parent(self): @@ -171,18 +171,12 @@ def is_folder(self): def get_children(self): """Return the children of this folder""" - try: - children = os.listdir(self) - except OSError: - return [] + children = self.project.fscommands.listdir(self.real_path) result = [] for name in children: - try: - child = self.get_child(name) - except exceptions.ResourceNotFoundError: - continue + child = self.get_child(name) if not self.project.is_ignored(child): - result.append(self.get_child(name)) + result.append(child) return result def create_file(self, file_name): @@ -257,7 +251,7 @@ def _add_pattern(self, pattern): re_pattern = "^(.*/)?" + re_pattern + "(/.*)?$" self.compiled_patterns.append(re.compile(re_pattern)) - def does_match(self, resource): + def does_match(self, resource: Resource): for pattern in self.compiled_patterns: if pattern.match(resource.path): return True diff --git a/rope/base/simplify.py b/rope/base/simplify.py index 3198c26de..b2a58960c 100644 --- a/rope/base/simplify.py +++ b/rope/base/simplify.py @@ -3,7 +3,10 @@ This module is here to help source code analysis. """ +import io import re +import sys +import tokenize from rope.base import codeanalyze, utils @@ -23,16 +26,25 @@ def real_code(source): only in offsets. """ collector = codeanalyze.ChangeCollector(source) + has_fstring = False for start, end, matchgroups in ignored_regions(source): if source[start] == "#": replacement = " " * (end - start) elif "f" in matchgroups.get("prefix", "").lower(): replacement = None + has_fstring = True else: replacement = '"%s"' % (" " * (end - start - 2)) if replacement is not None: collector.add_change(start, end, replacement) source = collector.get_changed() or source + if has_fstring: + # f-strings are left untouched above so their expression parts keep + # their real offsets, but that leaves any literal or format-spec + # `()[]{}` text (e.g. the `[` in `f"[{x}"`) sitting in `source`, + # where the `_parens` pass below would miscount it as a real + # unmatched bracket and corrupt every following offset. Blank it. + source = _blank_fstring_literal_brackets(source) collector = codeanalyze.ChangeCollector(source) parens = 0 for match in _parens.finditer(source): @@ -48,6 +60,60 @@ def real_code(source): return source.replace("\\\n", " ").replace("\t", " ").replace(";", "\n") +def _blank_fstring_literal_brackets(source): + """Blank `()[]{}` chars that are f-string literal/format-spec text. + + Blanked characters become a single space each, so length and every + character offset are preserved exactly. + + Approach: tokenize `source` and protect every character covered by an + `OP` token. A `()[]{}` character that no `OP` token covers cannot be + real syntax -- comments and non-f strings are already blanked, so the + only place it can occur is inside an f-string's literal or format-spec + text. Real expression brackets living inside an f-string (e.g. the + `[`/`]` in `f"{[1,2][0]}"`, or `{width}` in `f"{x:{width}}"`) are `OP` + tokens, so they stay untouched. A bracket *character* that is literal + data inside a nested string (the `}` in `f"{d['}']}"`) is not an `OP` + token, so it is blanked to a space -- correct, since it is data, not a + real bracket, and must not sway the `_parens` continuation count. This + also handles `{{`/`}}` escapes: tokenize covers the doubled brace with + a single FSTRING_MIDDLE token, leaving the escaping second brace under + no token span -- and, not being an `OP` either, the complement-of-OP + rule correctly blanks it. + + Only Python 3.12+ can make this distinction (PEP 701 added + FSTRING_START/MIDDLE/END; earlier an f-string is one opaque STRING + token). On earlier versions this is a no-op and the pre-existing + behaviour is unchanged. + """ + if sys.version_info < (3, 12): + return source + try: + tokens = list(tokenize.generate_tokens(io.StringIO(source).readline)) + except (tokenize.TokenError, SyntaxError, IndentationError, ValueError): + # `source` may be incomplete/invalid mid-edit (e.g. an unterminated + # f-string or an unbalanced paren -- exactly what the `_parens` pass + # handles). Fall back to the pre-existing behaviour. + return source + + lines = codeanalyze.SourceLinesAdapter(source) + + def offset(position): + row, col = position + return lines.get_line_start(row) + col + + protected = set() + for tok in tokens: + if tok.type == tokenize.OP: + protected.update(range(offset(tok.start), offset(tok.end))) + + collector = codeanalyze.ChangeCollector(source) + for i, c in enumerate(source): + if c in "()[]{}" and i not in protected: + collector.add_change(i, i + 1, " ") + return collector.get_changed() or source + + @utils.cached(7) def ignored_regions(source): """Return ignored regions like strings and comments in `source`""" diff --git a/rope/base/sourcebindings.py b/rope/base/sourcebindings.py new file mode 100644 index 000000000..88f1225f1 --- /dev/null +++ b/rope/base/sourcebindings.py @@ -0,0 +1,2582 @@ +"""Canonical source identities for expression scopes and function headers. + +The index is built solely from the module AST. Structural scopes are resolved +only after construction, so looking up a decorator cannot recursively build a +partially published index. Source identities never contain call-time values. +""" + +import sys +import typing +from dataclasses import dataclass +from operator import attrgetter + +from rope.base import ( + ast, + codeanalyze, + exceptions, + nameanalyze, + pynames, + pyobjects, + pyscopes, +) + +_TYPE_ALIAS_TYPES = getattr(ast, "TypeAlias", ()) + +_NO_LINENO = object() + +# Walk dispatch: node classes are classified once into the branch that +# ``_visit`` would take, so the per-node ``isinstance`` chain collapses into a +# dictionary lookup. Subclasses fall back to the chain and are cached too. +_KIND_OTHER = 0 +_KIND_FUNCTION = 1 +_KIND_TYPEALIAS = 2 +_KIND_CLASS = 3 +_KIND_LAMBDA = 4 +_KIND_COMPREHENSION = 5 +_KIND_WALRUS = 6 + +_NODE_KINDS = {} + +# Per node class, the fields whose values the generic walk must descend into: +# ``(field name, holds a list)``. Computed from the first instance seen and +# sound for every instance of the class: scalar fields enter the spec only when +# they hold ``None`` or an AST (``_visit`` skips other values exactly like +# ``ast.iter_child_nodes``), list fields keep the per-item AST filter. +_CHILD_SPECS = {} + + +def _classify_node(node): + kind = _KIND_OTHER + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + kind = _KIND_FUNCTION + elif _TYPE_ALIAS_TYPES and isinstance(node, _TYPE_ALIAS_TYPES): + kind = _KIND_TYPEALIAS + elif isinstance(node, ast.ClassDef): + kind = _KIND_CLASS + elif isinstance(node, ast.Lambda): + kind = _KIND_LAMBDA + elif isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)): + kind = _KIND_COMPREHENSION + elif isinstance(node, ast.NamedExpr): + kind = _KIND_WALRUS + _NODE_KINDS[node.__class__] = kind + return kind + + +def _child_spec(node): + spec = _CHILD_SPECS.get(node.__class__) + if spec is None: + fields = [] + for name in node._fields: + try: + value = getattr(node, name) + except AttributeError: + continue + if value.__class__ is list: + fields.append((name, True)) + elif value is None or isinstance(value, ast.AST): + fields.append((name, False)) + spec = tuple(fields) + _CHILD_SPECS[node.__class__] = spec + return spec + + +def _resolved(environment): + if isinstance(environment, _StructuralReference): + return environment.resolve() + return environment + + +class _StructuralReference: + """An AST owner, without forcing Rope's structural visitor during build.""" + + def __init__(self, index, node, parent=None, entry=None): + self.index = index + self.node = node + self.parent = parent + self.entry = parent if entry is None else entry + self._scope = None + self.header_slots = {} + self.redirected_headers = {} + self._installed_slots = False + self._installing_slots = False + + def resolve(self): + if self._scope is None: + if self.parent is None: + self._scope = self.index.pymodule.get_scope() + else: + parent_scope = self.parent.resolve() + for scope in parent_scope.get_scopes(): + if scope.pyobject.get_ast() is self.node: + self._scope = scope + break + if self._scope is None: + raise RuntimeError("source owner has no structural scope") + if ( + (self.header_slots or self.redirected_headers) + and not self._installed_slots + and not self._installing_slots + ): + self._installing_slots = True + try: + self._install_header_slots() + self._installed_slots = True + finally: + self._installing_slots = False + return self._scope + + def _install_header_slots(self): + from rope.base import pynamesdef + + names = self._scope.get_names() + for name, assignments in self.header_slots.items(): + binding = names.get(name) + if not isinstance(binding, (pynames.AssignedName, pynames.ParameterName)): + binding = pynamesdef.AssignedName( + assignments[0].get_lineno(), self.index.pymodule + ) + names[name] = binding + if self.get_kind() in ("Module", "Class"): + self._scope.pyobject._get_structural_attributes()[name] = binding + attributes = self._scope.pyobject.attributes.get() + if attributes is not None: + attributes[name] = binding + binding.binding_scope = self._scope + value_binding = binding + if isinstance(binding, pynamesdef.ParameterName): + if binding.assigned_value is None: + binding.assigned_value = pynamesdef.AssignedName( + module=self.index.pymodule + ) + binding.assigned_value.initial_binding = binding + value_binding = binding.assigned_value + if isinstance(value_binding, pynames.AssignedName): + present = { + id(assignment.ast_node) for assignment in value_binding.assignments + } + value_binding.assignments.extend( + assignment + for assignment in assignments + if id(assignment.ast_node) not in present + ) + if isinstance(binding, pynamesdef.ParameterName): + value_binding.assignments.sort( + key=lambda assignment: self.index._span( + assignment.ast_node or assignment.type_hint + )[0] + ) + value_binding.invalidate() + for name, owner in self.redirected_headers.items(): + names[name] = owner.resolve().get_names()[name] + + def lookup(self, name): + return self.resolve().lookup(name) + + def _propagated_lookup(self, name): + return self.resolve()._propagated_lookup(name) + + def get_names(self): + return self.resolve().get_names() + + def get_defined_names(self): + return self.resolve().get_defined_names() + + def get_propagated_names(self): + return self.resolve().get_propagated_names() + + def get_module(self): + return self.index.pymodule + + def get_region(self): + return self.resolve().get_region() + + def __getattr__(self, name): + return getattr(self.resolve(), name) + + def get_kind(self): + if isinstance(self.node, ast.ClassDef): + return "Class" + if isinstance(self.node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return "Function" + return "Module" + + @property + def pyobject(self): + return self.resolve().pyobject + + +class _PropagationView: + """Use an enclosing environment without inheriting ordinary class names.""" + + def __init__(self, environment): + self.environment = environment + self.parent = environment.parent + + def lookup(self, name): + return self.environment._propagated_lookup(name) + + def _propagated_lookup(self, name): + return self.lookup(name) + + def get_names(self): + return self.environment.get_propagated_names() + + def get_propagated_names(self): + return self.get_names() + + def __getattr__(self, name): + return getattr(self.environment, name) + + +@dataclass(frozen=True) +class AnnotationPolicy: + mode: str + runtime_profile: tuple + resolver_contract: str + + +@dataclass(frozen=True) +class SourceClassLayout: + """Source regions and native annotation bindings of one class declaration.""" + + region: tuple[int, int] + header: tuple[int, int] + header_end: int + body: tuple[int, int] + name_region: tuple[int, int] + bases_insertion_offset: int + arguments_region: tuple[int, int] | None + keywords: tuple[tuple[int, int], ...] + bases: tuple[tuple[int, int], ...] + base_bindings: tuple[tuple[tuple[int, int], pynames.PyName | None], ...] + decorators: tuple[tuple[int, int], ...] + type_parameters: tuple[tuple[str, tuple[int, int]], ...] + annotated_bindings: tuple[tuple[tuple[int, int], pynames.PyName | None, bool], ...] + + +@dataclass(frozen=True) +class SourceAssignmentLayout: + """Native assignment identity, regions and original value binding.""" + + kind: str + region: tuple[int, int] + targets: tuple[tuple[str, tuple[int, int]], ...] + value: tuple[int, int] | None + annotation: tuple[int, int] | None + value_binding: pynames.PyName | None + root_binding: pynames.PyName | None + string_sequence: tuple[str, ...] | None + type_parameters: tuple[tuple[str, tuple[int, int]], ...] + + +@dataclass(frozen=True) +class SourceImportLayout: + """Existing import declarations and lexical provider/alias regions.""" + + region: tuple[int, int] + module_region: tuple[int, int] | None + module_name: str | None + level: int + aliases: tuple[tuple[str, str | None, tuple[int, int]], ...] + separators: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class SourceStringMapping: + """Literal mapping tokens and wrapper provenance from an indexed call.""" + + region: tuple[int, int] + callee_binding: pynames.PyName | None + wrapper_bindings: tuple[pynames.PyName | None, ...] + entries: tuple[tuple[str, tuple[str, ...], tuple[tuple[int, int], ...]], ...] | None + + +class SourceContext: + """A recorded node environment (hand-rolled frozen dataclass equivalent). + + The index records one context per visited node, so the class keeps the + dataclass public surface (fields, defaults, equality, hash, repr) while + avoiding the generated frozen ``__init__`` on the hot build path. + """ + + __slots__ = ( + "_environment", + "_structural_scope", + "source_node", + "role", + "annotation_owner", + "annotation_policy", + "branch_context", + ) + + def __init__( + self, + _environment, + _structural_scope, + source_node, + role, + annotation_owner=None, + annotation_policy=None, + branch_context=(), + ): + self._environment = _environment + self._structural_scope = _structural_scope + self.source_node = source_node + self.role = role + self.annotation_owner = annotation_owner + self.annotation_policy = annotation_policy + self.branch_context = branch_context + + @property + def environment(self): + return _resolved(self._environment) + + @property + def structural_scope(self): + return _resolved(self._structural_scope) + + @property + def access(self): + context = getattr(self.source_node, "ctx", None) + if isinstance(context, ast.Load): + return "read" + if isinstance(context, ast.Store): + return "write" + if isinstance(context, ast.Del): + return "delete" + return None + + def __eq__(self, other): + if other.__class__ is self.__class__: + return ( + self._environment, + self._structural_scope, + self.source_node, + self.role, + self.annotation_owner, + self.annotation_policy, + self.branch_context, + ) == ( + other._environment, + other._structural_scope, + other.source_node, + other.role, + other.annotation_owner, + other.annotation_policy, + other.branch_context, + ) + return NotImplemented + + def __hash__(self): + return hash( + ( + self._environment, + self._structural_scope, + self.source_node, + self.role, + self.annotation_owner, + self.annotation_policy, + self.branch_context, + ) + ) + + def __repr__(self): + return ( + f"SourceContext(_environment={self._environment!r}, " + f"_structural_scope={self._structural_scope!r}, " + f"source_node={self.source_node!r}, role={self.role!r}, " + f"annotation_owner={self.annotation_owner!r}, " + f"annotation_policy={self.annotation_policy!r}, " + f"branch_context={self.branch_context!r})" + ) + + +class SourceFragment: + """A recorded node span (hand-rolled frozen dataclass equivalent).""" + + __slots__ = ("start", "end", "depth", "context") + + def __init__(self, start, end, depth, context): + self.start = start + self.end = end + self.depth = depth + self.context = context + + def __eq__(self, other): + if other.__class__ is self.__class__: + return ( + self.start, + self.end, + self.depth, + self.context, + ) == (other.start, other.end, other.depth, other.context) + return NotImplemented + + def __hash__(self): + return hash((self.start, self.end, self.depth, self.context)) + + def __repr__(self): + return ( + f"SourceFragment(start={self.start!r}, end={self.end!r}, " + f"depth={self.depth!r}, context={self.context!r})" + ) + + +class SourceLocalName(pynames.AssignedName): + def __init__(self, environment, name, node): + self.environment = environment + self.binding_scope = environment + self.name = name + self.node = node + self.assignments = [] + self._value_name = None + + def get_object(self, *, branch_context=()): + if self.assignments: + if self._value_name is None: + from rope.base import pynamesdef + + self._value_name = pynamesdef.AssignedName( + self.node.lineno, self.environment.get_module() + ) + self._value_name.assignments.extend(self.assignments) + return self._value_name.get_object(branch_context=branch_context) + return pyobjects.get_unknown() + + def get_definition_location(self): + return self.environment.get_module(), self.node.lineno + + def invalidate(self): + if self._value_name is not None: + self._value_name.invalidate() + + +class LambdaParameterName(pynames.ParameterName): + def __init__(self, pyfunction, name, index, node): + self.pyfunction = pyfunction + self.binding_scope = pyfunction.get_scope() + self.name = name + self.index = index + self.node = node + + def get_object(self): + return pyobjects.get_unknown() + + def get_objects(self): + return [] + + def get_definition_location(self): + return self.pyfunction.get_module(), self.node.lineno + + +class TypeParameterName(SourceLocalName): + pass + + +class SourceValueName(pynames.PyName): + """An attribute's call value, retaining its canonical source name.""" + + def __init__(self, source_pyname, pyobject): + self.source_pyname = source_pyname + self.pyobject = pyobject + + def get_object(self): + return self.pyobject + + def get_definition_location(self): + return self.source_pyname.get_definition_location() + + def __getattr__(self, name): + return getattr(self.source_pyname, name) + + +def source_lambda_attribute_value(primary, attribute): + """Separate an initializer's lambda value from the class namespace value.""" + from rope.base import evaluate + + receiver = primary.get_object() + class_access = isinstance(receiver, pyobjects.AbstractClass) + receiver_class = receiver if class_access else receiver.get_type() + if not isinstance(receiver_class, pyobjects.PyClass): + return None + module, _ = attribute.get_definition_location() + if module is None: + return None + index = module._get_source_binding_index() + storage = {"class": [], "instance": []} + for assignment in getattr(attribute, "assignments", ()): + context = index.context_for_node(assignment.ast_node) + if context is None or assignment.levels: + continue + owner = context.structural_scope + if owner.get_kind() == "Class" and owner.pyobject is receiver_class: + kind = "class" + elif ( + owner.get_kind() == "Function" + and owner.pyobject.parent is receiver_class + and owner.pyobject.get_name() == "__init__" + and owner.pyobject.get_kind() == "method" + ): + kind = "instance" + else: + continue + if any( + isinstance(statement, (ast.Assign, ast.AnnAssign)) + and statement.value is assignment.ast_node + for statement in owner.pyobject.get_ast().body + ): + storage[kind].append((assignment, context)) + if not storage["class"] or not storage["instance"]: + return None + assignment, context = storage["class" if class_access else "instance"][-1] + value = evaluate.eval_node(context.environment, assignment.ast_node) + function = value.get_object() if value is not None else None + if isinstance(function, SourceLambdaFunction): + return SourceValueName(attribute, function) + return None + + +class OpaqueSourceName(pynames.PyName): + is_refactoring_identity_safe = False + + def __init__(self, environment, name, node, candidates): + self.environment = environment + self.name = name + self.node = node + self.reason = "generic annotation resolver identity is not established" + self.candidate_environments = tuple(candidates) + + def get_object(self): + return pyobjects.get_unknown() + + def get_definition_location(self): + return self.environment.get_module(), self.node.lineno + + @property + def candidate_binders(self): + result = [] + for environment in self.candidate_environments: + candidate = environment.lookup(self.name) + if candidate is not None and candidate is not self: + if ( + not isinstance(candidate, OpaqueSourceName) + and candidate not in result + ): + result.append(candidate) + return tuple(result) + + def may_reference(self, pyname): + from rope.refactor.occurrences import same_pyname + + return any( + same_pyname(candidate, pyname) for candidate in self.candidate_binders + ) + + +@dataclass(frozen=True) +class OpaqueReference: + node: object + context: SourceContext + pyname: OpaqueSourceName + + def may_reference(self, pyname): + return self.pyname.may_reference(pyname) + + +class _SourceObject(pyobjects.PyObject): + def __init__(self, environment): + super().__init__(pyobjects.get_base_type("Unknown")) + self.environment = environment + + def get_module(self): + return self.environment.get_module() + + def get_scope(self): + return self.environment + + def get_ast(self): + return self.environment.node + + def get_attributes(self): + return self.environment.get_names() + + def get_kind(self): + return self.environment.get_kind() + + +class BindingEnvironment(pyscopes.Scope): + def __init__(self, index, node, parent, structural_scope, kind): + self.index = index + self.node = node + self.structural_reference = structural_scope + self.kind = kind + self.names = {} + self.children = [] + self.capture_parent = parent + super().__init__(index.pymodule.pycore, _SourceObject(self), parent) + + @property + def structural_scope(self): + return _resolved(self.structural_reference) + + def get_module(self): + return self.index.pymodule + + def get_names(self): + return self.names + + def get_defined_names(self): + return self.names + + def _create_scopes(self): + return [] + + def get_binding_scopes(self): + return self.index.children_of(self) + + def get_kind(self): + return self.kind + + def get_start(self): + return self.node.lineno + + def get_end(self): + return getattr(self.node, "end_lineno", self.node.lineno) + + def get_body_start(self): + return self.get_start() + + def get_logical_end(self): + return self.get_end() + + def get_region(self): + return self.index._span(self.node) + + def _get_global_scope(self): + return self.index.pymodule.get_scope() + + start = property(get_start) + end = property(get_end) + logical_end = property(get_logical_end) + + +class AnnotationEnvironment(BindingEnvironment): + def __init__( + self, index, node, entry, owner, policy, generic_names, candidates=None + ): + super().__init__(index, node, entry, owner, "Annotation") + self.policy = policy + self.capture_parent = index._capture(entry) + self.generic_names = generic_names + self.candidates = candidates + self.opaque_names = {} + + def lookup(self, name): + if name in self.generic_names: + if name not in self.opaque_names: + candidates = self.candidates or self.index._candidate_environments( + self.parent + ) + self.opaque_names[name] = OpaqueSourceName( + self, name, self.node, candidates + ) + return self.opaque_names[name] + class_reference = self.parent + if isinstance(class_reference, _GenericOwnerEnvironment): + class_reference = class_reference.structural_reference + if ( + self.policy.mode in ("eager", "lazy") + and isinstance(class_reference, _StructuralReference) + and class_reference.get_kind() == "Class" + ): + cutoff = self.node if self.policy.mode == "eager" else None + status = self.index._class_binding_before( + class_reference.node, name, cutoff + ) + if status == "absent": + return self.index._class_fallback(class_reference, name).lookup(name) + if status == "ambiguous": + if name not in self.opaque_names: + fallback = self.index._class_fallback(class_reference, name) + self.opaque_names[name] = OpaqueSourceName( + self, name, self.node, (class_reference, fallback) + ) + return self.opaque_names[name] + return self.parent.lookup(name) + + def _propagated_lookup(self, name): + return self.lookup(name) + + def get_names(self): + names = set(self.generic_names) + environment = self.parent + direct = True + while environment is not None: + visible = ( + environment.get_names() + if direct + else environment.get_propagated_names() + ) + names.update(visible) + if not isinstance(environment, TypeParameterEnvironment): + direct = False + environment = environment.parent + result = {} + for name in names: + binding = self.lookup(name) + if binding is not None: + result[name] = binding + return result + + def get_propagated_names(self): + return self.get_names() + + +class TypeAliasEnvironment(AnnotationEnvironment): + """A lazy alias resolver with its own canonical type parameters.""" + + def __init__(self, index, node, entry, generic, owner): + super().__init__( + index, + node, + generic, + owner, + index.alias_policy, + index._generic_names(generic), + ) + self.entry_environment = entry + + def _class_reference(self): + entry = self.entry_environment + if isinstance(entry, _GenericOwnerEnvironment) and entry.kind == "Class": + entry = entry.structural_reference + if isinstance(entry, _StructuralReference) and entry.get_kind() == "Class": + return entry + return None + + def _parameter_environment(self, name): + environment = self.parent + while environment is not None: + if ( + isinstance(environment, TypeParameterEnvironment) + and name in environment.names + ): + return environment + environment = environment.parent + return None + + def opaque_binding(self, name): + class_reference = self._class_reference() + if class_reference is None: + return None + status = self.index._class_binding_before(class_reference.node, name, None) + parameter = self._parameter_environment(name) + if parameter is not None and status != "absent": + candidates = (class_reference, parameter) + elif parameter is None and status == "ambiguous": + candidates = ( + class_reference, + self.index._class_fallback(class_reference, name), + ) + else: + return None + if name not in self.opaque_names: + self.opaque_names[name] = OpaqueSourceName( + self, name, self.node, candidates + ) + return self.opaque_names[name] + + def lookup(self, name): + opaque = self.opaque_binding(name) + if opaque is not None: + return opaque + class_reference = self._class_reference() + if class_reference is not None and name not in self.generic_names: + status = self.index._class_binding_before(class_reference.node, name, None) + if status == "absent": + return self.index._class_fallback(class_reference, name).lookup(name) + return self.parent.lookup(name) + + +class ComprehensionBindingScope(BindingEnvironment): + pass + + +class LambdaBindingScope(BindingEnvironment): + pass + + +class TypeParameterEnvironment(BindingEnvironment): + def lookup(self, name): + return self.names[name] if name in self.names else self.parent.lookup(name) + + +class _LambdaArgumentValues: + """Match values locally without changing the ordinary Arguments protocol.""" + + def __init__(self, function, receiver): + self.function = function + self.receiver_name = ( + function.positional[0] + if receiver is not None and function.positional + else None + ) + + def match(self, argument_nodes, argument_scope): + from rope.base import evaluate + + values = {} + positional = [ + name for name in self.function.positional if name != self.receiver_name + ] + keyword_names = set(self.function.keywords) - {self.receiver_name} + next_position = 0 + expanded_position = False + expanded_keyword = False + keyword_supplied = set() + conflicts = set() + for node in argument_nodes: + if isinstance(node, ast.Starred): + expanded_position = True + elif isinstance(node, ast.keyword): + if node.arg is None: + expanded_keyword = True + elif node.arg in keyword_names: + if node.arg in values: + conflicts.add(node.arg) + keyword_supplied.add(node.arg) + values[node.arg] = evaluate.eval_node(argument_scope, node.value) + elif not expanded_position: + if next_position < len(positional): + values[positional[next_position]] = evaluate.eval_node( + argument_scope, node + ) + next_position += 1 + for name in tuple(values): + if ( + name in conflicts + or (expanded_keyword and name in keyword_names) + or ( + expanded_position + and name in positional + and name in keyword_supplied + ) + ): + values[name] = None + for name, expression in self.function.defaults.items(): + if name in values or name == self.receiver_name: + continue + if expanded_position and name in positional: + continue + if expanded_keyword and name in keyword_names: + continue + values[name] = evaluate.eval_node( + _resolved(self.function.creation_scope), expression + ) + return values + + +class SourceLambdaFunction(pyobjects.AbstractFunction): + def __init__(self, index, node, creation_scope, scope): + super().__init__() + self.index = index + self.node = node + self.creation_scope = creation_scope + self.source_function = self + self.bound_creation_scope = None + self.bound_receiver = None + self.descriptor_kind = "function" + self.scope = scope + args = node.args + self.posonly = [arg.arg for arg in args.posonlyargs] + self.positional = self.posonly + [arg.arg for arg in args.args] + self.keywords = [arg.arg for arg in args.args + args.kwonlyargs] + self.param_names = self.positional + [arg.arg for arg in args.kwonlyargs] + if args.vararg is not None: + self.param_names.append(args.vararg.arg) + if args.kwarg is not None: + self.param_names.append(args.kwarg.arg) + self.defaults = dict( + zip( + self.positional[len(self.positional) - len(args.defaults) :], + args.defaults, + ) + ) + self.defaults.update( + (arg.arg, default) + for arg, default in zip(args.kwonlyargs, args.kw_defaults) + if default is not None + ) + + def get_parameters(self): + return {name: self.scope.names[name] for name in self.param_names} + + def bind_creation_scope(self, scope): + bound = object.__new__(SourceLambdaFunction) + bound.__dict__ = dict(self.__dict__) + bound.source_function = self.source_function + bound.bound_creation_scope = scope + bound.creation_scope = scope + return bound + + def as_descriptor(self, kind): + descriptor = object.__new__(SourceLambdaFunction) + descriptor.__dict__ = dict(self.__dict__) + descriptor.descriptor_kind = kind + return descriptor + + def bind_receiver(self, receiver): + bound = object.__new__(SourceLambdaFunction) + bound.__dict__ = dict(self.__dict__) + bound.bound_receiver = receiver + return bound + + def descriptor_receiver(self, primary, attribute): + from rope.base import evaluate + + if self.bound_receiver is not None: + return None + receiver = primary.get_object() + receiver_class = ( + receiver + if isinstance(receiver, pyobjects.AbstractClass) + else receiver.get_type() + ) + if not isinstance(receiver_class, pyobjects.PyClass): + return None + module, _ = attribute.get_definition_location() + if module is None: + return None + class_storage = False + for assignment in getattr(attribute, "assignments", ()): + context = module._get_source_binding_index().context_for_node( + assignment.ast_node + ) + if context is None or context.structural_scope.get_kind() != "Class": + continue + value = evaluate.eval_node(context.environment, assignment.ast_node) + assigned = value.get_object() if value is not None else None + if ( + isinstance(assigned, SourceLambdaFunction) + and assigned.source_function is self.source_function + ): + class_storage = True + break + if not class_storage or self.descriptor_kind == "staticmethod": + return None + if self.descriptor_kind == "classmethod": + return pynames.UnboundName(receiver_class) + if not isinstance(receiver, pyobjects.AbstractClass): + return primary + return None + + @property + def parent(self): + return self.scope.structural_scope.pyobject + + def get_param_names(self, special_args=True): + if not special_args: + return self.positional + [arg.arg for arg in self.node.args.kwonlyargs] + return list(self.param_names) + + def get_positional_param_names(self): + return list(self.positional) + + def get_keyword_param_names(self): + return list(self.keywords) + + def get_parameter_defaults(self): + return dict(self.defaults) + + def get_scope(self): + return self.scope + + def get_module(self): + return self.index.pymodule + + def get_ast(self): + return self.node + + def get_name(self): + return "lambda" + + def get_kind(self): + return "lambda" + + def get_attributes(self): + return {} + + def get_returned_object(self, arguments): + from rope.base import arguments as argument_matching + from rope.base import evaluate + + values = {} + receiver = self.bound_receiver + if isinstance(arguments, argument_matching.MixedArguments): + if receiver is None: + receiver = arguments.pyname + arguments = arguments.args + if arguments is not None and hasattr(arguments, "args"): + values.update( + _LambdaArgumentValues(self, receiver).match( + arguments.args, arguments.scope + ) + ) + elif arguments is not None: + names = ( + self.param_names[1:] + if receiver is not None and self.positional + else self.param_names + ) + values.update(zip(names, arguments.get_pynames(names))) + else: + values.update( + _LambdaArgumentValues(self, receiver).match( + [], _resolved(self.creation_scope) + ) + ) + if receiver is not None and self.positional: + values[self.positional[0]] = receiver + for name in self.param_names: + if values.get(name) is None: + values[name] = pynames.UnboundName() + capture = self.bound_creation_scope + if capture is None: + capture = _resolved(self.scope.parent) + overlay = evaluate.SourceValueScope(self.scope, capture, values) + result = evaluate.eval_node(overlay, self.node.body) + return result.get_object() if result is not None else pyobjects.get_unknown() + + +class SourceBindingIndex: + def __init__(self, pymodule, *, _defer_bodies=False): + self.pymodule = pymodule + self._defer_bodies = _defer_bodies + self._pending_bodies = {} + self._attribute_write_index = None + self._name_write_index = None + self.source = pymodule.source_code + self.fragments = [] + self._fragment_positions = {} + self.node_contexts = {} + self._node_ids = None + self._branch_context = () + self.token_bindings = [] + self.source_callables = {} + self.comprehensions = {} + self.environments = [] + self._opaque_references = [] + # Contexts whose node can be an opaque annotation reference, keyed by + # node id (mirroring node_contexts). Recorded during the walk so + # `opaque_references` scans only these instead of every context. + self._annotation_name_contexts = {} + self.lines = codeanalyze.ASTLinesAdapter(self.source) + tree = pymodule.get_ast() + self.global_reference = _StructuralReference(self, tree) + self.owner_references = [self.global_reference] + self._owners_by_node = {id(tree): self.global_reference} + self._preparing_headers = False + self._class_symbols = {} + self.future_annotations = any( + isinstance(node, ast.ImportFrom) + and node.module == "__future__" + and any(alias.name == "annotations" for alias in node.names) + for node in tree.body + ) + mode = ( + "future-string" + if self.future_annotations + else "lazy" if sys.version_info >= (3, 14) else "eager" + ) + self.policy = AnnotationPolicy( + mode, + sys.version_info[:2], + "default-function-hints" if self.future_annotations else "source-value", + ) + self.alias_policy = AnnotationPolicy( + "lazy", sys.version_info[:2], "source-value" + ) + self.state = "BUILDING" + self._visit(tree, self.global_reference, self.global_reference, "module", 0) + if not self._pending_bodies: + self._sort_fragments() + self.state = "FROZEN" + + def attribute_writes(self, name): + """Return matching stores/deletions from this immutable source snapshot.""" + if self._attribute_write_index is None: + index = {} + for node in ast.walk(self.global_reference.node): + if isinstance(node, ast.Attribute) and isinstance( + node.ctx, (ast.Store, ast.Del) + ): + index.setdefault(node.attr, []).append(node) + self._attribute_write_index = { + key: tuple(nodes) for key, nodes in index.items() + } + return self._attribute_write_index.get(name, ()) + + def name_writes(self, pyname, name): + """Return the stores/deletions of `name` that write the `pyname` binding + + A write in another scope counts when it resolves to the same binding, + as under a ``global`` or ``nonlocal`` declaration. + """ + if self._name_write_index is None: + index = {} + for node in ast.walk(self.global_reference.node): + if isinstance(node, ast.Name) and isinstance( + node.ctx, (ast.Store, ast.Del) + ): + index.setdefault(node.id, []).append(node) + self._name_write_index = index + return tuple( + node + for node in self._name_write_index.get(name, ()) + if self.context_for_node(node).environment.lookup(name) is pyname + ) + + def _sort_fragments(self): + self.fragments.sort(key=attrgetter("start", "end", "depth")) + self._fragment_positions = { + id(fragment.context.source_node): position + for position, fragment in enumerate(self.fragments) + } + + def _complete(self): + if not self._defer_bodies: + return self + self._defer_bodies = False + while self._pending_bodies: + self._materialize_body(next(iter(self._pending_bodies))) + self._sort_fragments() + return self + + def _materialize_body(self, key): + node, environment, owner, depth = self._pending_bodies.pop(key) + previous = self._branch_context + self._branch_context = () + try: + for statement in node.body: + self._visit(statement, environment, owner, "function-body", depth + 1) + finally: + self._branch_context = previous + + def _materialize_at(self, offset): + # Completing an outer body discovers nested declarations in the same + # index; descend only into the body containing the requested source. + while True: + for key, (node, _, _, _) in self._pending_bodies.items(): + first = node.body[0] + start = min( + self._span(part)[0] + for part in [first, *getattr(first, "decorator_list", ())] + ) + _, end = self._span(node) + if start <= offset < end: + self._materialize_body(key) + break + else: + return + + def _span(self, node): + if not hasattr(node, "lineno"): + return 0, len(self.source) + return self.lines[node] + + def _record(self, node, environment, owner, role, depth, annotation=None): + lineno = getattr(node, "lineno", _NO_LINENO) + if lineno is _NO_LINENO and not isinstance(node, ast.Module): + return None + if annotation is None: + context = SourceContext( + environment, owner, node, role, branch_context=self._branch_context + ) + else: + context = SourceContext( + environment, + owner, + node, + role, + annotation, + ( + self.alias_policy + if isinstance(annotation, _TYPE_ALIAS_TYPES) + else self.policy + ), + self._branch_context, + ) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + self._annotation_name_contexts[id(node)] = context + key = id(node) + self.node_contexts[key] = context + if lineno is not _NO_LINENO: + start, end = self.lines[node] + fragment = SourceFragment(start, end, depth, context) + position = self._fragment_positions.get(key) + if position is None: + self._fragment_positions[key] = len(self.fragments) + self.fragments.append(fragment) + else: + self.fragments[position] = fragment + return context + + def absolute_import_roots(self): + """Return absolute import roots from the existing declaration index.""" + roots = set() + for fragment in self.fragments: + node = fragment.context.source_node + if isinstance(node, ast.Import): + roots.update(alias.name.partition(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + roots.add(node.module.partition(".")[0]) + return frozenset(roots) + + def class_layout(self, scope): + """Read class layout and annotation origins from this indexed module.""" + from rope.base import evaluate, worder + + owner = scope.pyobject + if owner.get_module() is not self.pymodule: + raise ValueError("class layout scope belongs to another source module") + node = owner.get_ast() + if not isinstance(node, ast.ClassDef): + raise TypeError("class layout requires a source class scope") + start, end = self._span(node) + body_start, _ = self._span(node.body[0]) + leading_decorators = getattr(node.body[0], "decorator_list", ()) + if leading_decorators: + body_start = self.pymodule.lines.get_line_start( + min(item.lineno for item in leading_decorators) + ) + words = worder.Worder(self.source[start:body_start]) + name_offset = words.code_finder._find_first_non_space_char( + words.get_word_range(0)[1] + ) + name_start, name_end = words.get_word_range(name_offset) + type_parameters = getattr(node, "type_params", ()) + insertion = ( + words.get_word_parens_range(name_start, "[", "]")[1] + if type_parameters + else name_end + ) + next_token = words.code_finder._find_first_non_space_char(insertion) + arguments = None + if words.code_finder.code[next_token : next_token + 1] == "(": + first, last = words.get_word_parens_range(insertion - 1) + arguments = (start + first + 1, start + last - 1) + colon_start = arguments[1] - start + 1 if arguments is not None else insertion + colon = words.code_finder.code.index(":", colon_start) + annotations = [] + for member in node.body: + if not isinstance(member, ast.AnnAssign): + continue + annotation = member.annotation + qualifier = ( + annotation.value + if isinstance(annotation, ast.Subscript) + else annotation + ) + context = self.node_contexts[id(qualifier)] + annotations.append( + ( + self._span(member), + evaluate.eval_node(context.environment, qualifier), + member.value is not None, + ) + ) + return SourceClassLayout( + region=(start, end), + header=(start, body_start), + header_end=start + colon + 1, + body=(body_start, end), + name_region=(start + name_start, start + name_end), + bases_insertion_offset=start + insertion, + arguments_region=arguments, + keywords=tuple(self._span(keyword) for keyword in node.keywords), + bases=tuple(self._span(base) for base in node.bases), + base_bindings=tuple( + ( + self._span(base), + evaluate.eval_node(self.node_contexts[id(base)].environment, base), + ) + for base in node.bases + ), + decorators=tuple(self._span(item) for item in node.decorator_list), + type_parameters=tuple( + (parameter.name, self._span(parameter)) for parameter in type_parameters + ), + annotated_bindings=tuple(annotations), + ) + + def decorator_bindings(self, scope): + """Resolve decorator callees in their recorded source environments.""" + from rope.base import evaluate + + owner = scope.pyobject + if owner.get_module() is not self.pymodule: + raise ValueError("decorator scope belongs to another source module") + decorators = getattr(owner.get_ast(), "decorator_list", ()) + callees = ( + item.func if isinstance(item, ast.Call) else item for item in decorators + ) + return tuple( + evaluate.eval_node(self.node_contexts[id(callee)].environment, callee) + for callee in callees + ) + + def import_layouts(self): + """Expose indexed import boundaries without moving or reparsing them.""" + from rope.base import worder + + layouts = [] + for fragment in self.fragments: + node = fragment.context.source_node + if not isinstance(node, (ast.Import, ast.ImportFrom)): + continue + start, end = self._span(node) + words = worder.Worder(self.source[start:end]).code_finder + provider = None + if isinstance(node, ast.ImportFrom): + # The node already proves the grammar. Use Rope's lexical + # source to locate the provider before the import keyword. + keyword = words.code.rindex( + "import", 0, self._span(node.names[0])[0] - start + ) + first = words._find_first_non_space_char(4) + last = words._find_last_non_space_char(keyword - 1) + provider = (start + first, start + last + 1) + layouts.append( + SourceImportLayout( + (start, end), + provider, + node.module if isinstance(node, ast.ImportFrom) else None, + node.level if isinstance(node, ast.ImportFrom) else 0, + tuple( + (alias.name, alias.asname, self._span(alias)) + for alias in node.names + ), + tuple( + (start + comma, start + comma + 1) + for left, right in zip(node.names, node.names[1:]) + for comma in ( + words.code.index( + ",", + self._span(left)[1] - start, + self._span(right)[0] - start, + ), + ) + ), + ) + ) + return tuple(layouts) + + def string_mapping_arguments(self, callee, position): + """Read literal mapping arguments through the existing declaration index.""" + from rope.base import evaluate + + selected = [] + for fragment in self.fragments: + call = fragment.context.source_node + if not ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Name) + and call.func.id == callee + and 0 <= position < len(call.args) + ): + continue + value = call.args[position] + wrappers = [] + visited = set() + while id(value) not in visited: + visited.add(id(value)) + if isinstance(value, ast.Name): + binding = evaluate.eval_node( + self.node_contexts[id(value)].environment, value + ) + if isinstance(binding, SourceValueName): + binding = binding.source_pyname + if not isinstance(binding, pynames.AssignedName): + break + if len(binding.assignments) != 1: + break + assigned = binding.assignments[0].ast_node + if id(assigned) not in self.node_contexts: + break + value = assigned + continue + if ( + isinstance(value, ast.Call) + and len(value.args) == 1 + and not value.keywords + ): + wrappers.append( + evaluate.eval_node( + self.node_contexts[id(value.func)].environment, value.func + ) + ) + value = value.args[0] + continue + break + entries = [] + if isinstance(value, ast.Dict): + for key, item in zip(value.keys, value.values): + values = item.elts if isinstance(item, ast.Tuple) else (item,) + if not ( + isinstance(key, ast.Constant) + and isinstance(key.value, str) + and all( + isinstance(part, ast.Constant) + and isinstance(part.value, str) + for part in values + ) + ): + entries = None + break + entries.append( + ( + key.value, + tuple(part.value for part in values), + tuple(self._span(part) for part in values), + ) + ) + else: + entries = None + selected.append( + SourceStringMapping( + self._span(value), + evaluate.eval_node( + self.node_contexts[id(call.func)].environment, call.func + ), + tuple(wrappers), + None if entries is None else tuple(entries), + ) + ) + return tuple(selected) + + def assignment_layouts(self, scope=None, *, name=None, resolve_values=False): + """Return direct binding regions without reparsing the module source.""" + from rope.base import evaluate + + owner = self.pymodule if scope is None else scope.pyobject + if owner.get_module() is not self.pymodule: + raise ValueError("assignment scope belongs to another source module") + assignments = [] + for node in owner.get_ast().body: + if not ( + isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)) + or (_TYPE_ALIAS_TYPES and isinstance(node, _TYPE_ALIAS_TYPES)) + ): + continue + targets = ( + node.targets + if isinstance(node, ast.Assign) + else ( + (node.name,) + if _TYPE_ALIAS_TYPES and isinstance(node, _TYPE_ALIAS_TYPES) + else (node.target,) + ) + ) + selected = tuple( + (target.id, self._span(target)) + for target in targets + if isinstance(target, ast.Name) + ) + if name is not None and all(target != name for target, _ in selected): + continue + value = node.value + annotation = getattr(node, "annotation", None) + root = value + while isinstance(root, ast.Attribute): + root = root.value + assignments.append( + SourceAssignmentLayout( + kind=type(node).__name__, + region=self._span(node), + targets=selected, + value=self._span(value) if value is not None else None, + annotation=( + self._span(annotation) if annotation is not None else None + ), + value_binding=( + evaluate.eval_node( + self.node_contexts[id(value)].environment, value + ) + if resolve_values and value is not None + else None + ), + root_binding=( + evaluate.eval_node( + self.node_contexts[id(root)].environment, root + ) + if resolve_values and isinstance(root, ast.Name) + else None + ), + string_sequence=( + tuple(item.value for item in value.elts) + if isinstance(value, (ast.List, ast.Tuple, ast.Set)) + and all( + isinstance(item, ast.Constant) + and isinstance(item.value, str) + for item in value.elts + ) + else None + ), + type_parameters=tuple( + (parameter.name, self._span(parameter)) + for parameter in getattr(node, "type_params", ()) + ), + ) + ) + return tuple(assignments) + + @staticmethod + def _literal_strings(value): + """Return the strings of a literal list, tuple or set, else None.""" + if isinstance(value, (ast.List, ast.Tuple, ast.Set)) and all( + isinstance(item, ast.Constant) and isinstance(item.value, str) + for item in value.elts + ): + return tuple(item.value for item in value.elts) + return None + + def assignment_string_sequence(self, name, scope=None, *, class_metadata=False): + """Read a direct string-sequence declaration from the existing parse.""" + if scope is not None and class_metadata: + raise ValueError("Select either an explicit scope or class metadata") + if scope is not None and not isinstance( + scope, (pyscopes.GlobalScope, pyscopes.ClassScope) + ): + raise TypeError("Assignment query requires a module or class scope") + if scope is not None and scope.pyobject.get_module() is not self.pymodule: + raise ValueError("Assignment scope belongs to another source module") + module_body = self.pymodule.get_ast().body + body = module_body if scope is None else scope.pyobject.get_ast().body + visited = set() + declared = False + suffix = () + before = None + forward = False + while (id(body), name, before) not in visited: + # Python binds the last direct assignment; a later ``+=`` extends + # it and a bare annotation never rebinds an assigned name. + found = False + value = None + extensions = [] + for statement in body: + if ( + before is not None + and (statement.lineno, statement.col_offset) >= before + and found + ): + break + if isinstance(statement, ast.AugAssign): + if ( + found + and isinstance(statement.target, ast.Name) + and statement.target.id == name + ): + extensions.append(statement) + continue + if isinstance(statement, ast.Assign): + targets = statement.targets + elif isinstance(statement, ast.AnnAssign): + targets = (statement.target,) + else: + continue + if not any( + isinstance(target, ast.Name) and target.id == name + for target in targets + ) or (found and statement.value is None): + continue + found = True + declaration = statement + if ( + before is not None + and (statement.lineno, statement.col_offset) >= before + ): + forward = True + value = statement.value + extensions = [] + if not found: + if class_metadata and not declared: + class_metadata = False + classes = [ + node for node in module_body if isinstance(node, ast.ClassDef) + ] + if len(classes) == 1: + body = classes[0].body + continue + return () if declared else None + visited.add((id(body), name, before)) + declared = True + extended = () + for extension in extensions: + items = self._literal_strings(extension.value) + if not isinstance(extension.op, ast.Add) or items is None: + return () + extended += items + suffix = extended + suffix + if isinstance(value, (ast.List, ast.Tuple, ast.Set)): + items = self._literal_strings(value) + return () if forward or items is None else items + suffix + if ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and value.func.id in {"tuple", "list", "set", "frozenset"} + and len(value.args) == 1 + and not value.keywords + ): + value = value.args[0] + if not isinstance(value, ast.Name): + return () + name = value.id + # References read the binding at the declaration, not at the end + # of the module. Follow a forward-only reference only to report a + # cycle; it cannot supply a value. + before = declaration.lineno, declaration.col_offset + body = module_body + raise ValueError("Cyclic native string-sequence declaration: %s" % name) + + def context_at(self, offset): + self._materialize_at(offset) + matched = [item for item in self.fragments if item.start <= offset < item.end] + if matched: + return max( + matched, key=lambda item: (item.depth, -item.end + item.start) + ).context + scope = self.pymodule.get_scope().get_structural_scope_for_offset(offset) + return SourceContext(scope, scope, self.pymodule.get_ast(), "structure") + + def context_for_node(self, node): + if ( + id(node) not in self.node_contexts + and hasattr(node, "lineno") + and id(node) in self._module_node_ids() + ): + self._materialize_at(self._span(node)[0]) + return self.node_contexts.get(id(node)) + + def _module_node_ids(self): + # Why: a node from another AST (another module, or a consumer's own + # parse of this text) carries positions that do not belong to this + # source; it can never be recorded here, and spanning it reads + # arbitrary bytes, possibly inside a UTF-8 code point. + if self._node_ids is None: + self._node_ids = frozenset( + id(node) for node in ast.walk(self.pymodule.get_ast()) + ) + return self._node_ids + + def binding_for_token(self, offset): + self._materialize_at(offset) + for start, end, binding in self.token_bindings: + if start <= offset < end: + if isinstance(binding, tuple): + owner, name = binding + return owner.resolve().get_names().get(name) + return binding + return None + + def callable_for_node(self, node): + self.context_for_node(node) + return self.source_callables.get(id(node)) + + def comprehension_scope(self, node): + self.context_for_node(node) + return self.comprehensions.get(id(node)) + + def prepare_header_bindings(self, scope=None): + if self.state == "FROZEN" and not self._preparing_headers: + self._preparing_headers = True + try: + if scope is not None: + owner = self._owners_by_node.get(id(scope.pyobject.get_ast())) + if owner is not None and ( + owner.header_slots or owner.redirected_headers + ): + owner._scope = scope + owner.resolve() + else: + for owner in self.owner_references: + if ( + owner.header_slots or owner.redirected_headers + ) and owner.get_kind() in ("Module", "Class"): + owner.resolve() + finally: + self._preparing_headers = False + + def opaque_references(self): + known = {id(reference.node) for reference in self._opaque_references} + for context in self._annotation_name_contexts.values(): + node = context.source_node + if id(node) not in known: + binding = context.environment.lookup(node.id) + if isinstance(binding, OpaqueSourceName): + self._opaque_references.append( + OpaqueReference(node, context, binding) + ) + known.add(id(node)) + return tuple(self._opaque_references) + + def quoted_annotation_references(self, name): + """Index native type operands in quoted annotations, never literal values.""" + self._complete() + cache = getattr(self, "_quoted_reference_cache", None) + if cache is None: + cache = self._quoted_reference_cache = {} + if name in cache: + return cache[name] + roots = [] + for node in ast.walk(self.pymodule.get_ast()): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + arguments = ( + node.args.posonlyargs + node.args.args + node.args.kwonlyargs + ) + arguments += [ + arg + for arg in (node.args.vararg, node.args.kwarg) + if arg is not None + ] + roots.extend( + arg.annotation for arg in arguments if arg.annotation is not None + ) + if node.returns is not None: + roots.append(node.returns) + elif isinstance(node, ast.AnnAssign): + roots.append(node.annotation) + elif isinstance(node, _TYPE_ALIAS_TYPES): + roots.append(node.value) + references = [] + for node in roots: + if not any( + isinstance(part, ast.Constant) + and isinstance(part.value, str) + and name in part.value + for part in ast.walk(node) + ): + continue + context = self.context_for_node(node) + references.extend( + self._quoted_type_operands(node, self.source, None, context, name) + ) + cache[name] = tuple(references) + return cache[name] + + def _quoted_type_operands(self, node, source, coordinates, context, name): + from rope.base import builtins, evaluate + + if isinstance(node, ast.Constant) and isinstance(node.value, str): + if name not in node.value: + return + try: + expression = ast.parse(node.value, mode="eval").body + except SyntaxError: + return + positions = ( + self.lines + if coordinates is None + else codeanalyze.ASTLinesAdapter(source) + ) + start, end = positions[node] + token = source[start:end] + opening = next(index for index, char in enumerate(token) if char in "\"'") + quote = token[opening : opening + 3] + if quote not in ("'''", '"""'): + quote = token[opening] + prefix = token[: opening + len(quote)] + body = token[len(prefix) : -len(quote)] + mapped = [] + previous = "" + first = 0 + for cut in range(1, len(body) + 1): + try: + decoded = ast.literal_eval(prefix + body[:cut] + quote) + except SyntaxError: + continue + if not node.value.startswith(decoded) or len(decoded) <= len(previous): + continue + left, right = start + len(prefix) + first, start + len(prefix) + cut + physical = ( + (left, right) + if coordinates is None + else (coordinates[left][0], coordinates[right - 1][1]) + ) + mapped.extend([physical] * (len(decoded) - len(previous))) + previous, first = decoded, cut + if previous != node.value or len(mapped) != len(node.value): + raise exceptions.RefactoringError( + "Quoted type has no exact native literal source mapping." + ) + yield from self._quoted_type_operands( + expression, node.value, mapped, context, name + ) + return + if isinstance(node, (ast.Name, ast.Attribute)): + if ( + coordinates is not None + and (node.id if isinstance(node, ast.Name) else node.attr) == name + ): + positions = codeanalyze.ASTLinesAdapter(source) + first, last = positions[node] + word = last - len(name) + yield ( + coordinates[word][0], + coordinates[last - 1][1], + coordinates[first][0], + coordinates[last - 1][1], + context, + ast.unparse(node), + ) + if isinstance(node, ast.Attribute): + yield from self._quoted_type_operands( + node.value, source, coordinates, context, name + ) + return + if isinstance(node, ast.Subscript): + yield from self._quoted_type_operands( + node.value, source, coordinates, context, name + ) + binding = evaluate.eval_node(context.environment, node.value) + origin = None if binding is None else binding.get_object() + arguments = ( + node.slice.elts if isinstance(node.slice, ast.Tuple) else [node.slice] + ) + if builtins._is_runtime_form(origin, typing.Literal): + return + if builtins._is_runtime_class( + origin, typing.Annotated + ) or builtins._is_runtime_form(origin, typing.Annotated): + arguments = arguments[:1] + for argument in arguments: + yield from self._quoted_type_operands( + argument, source, coordinates, context, name + ) + elif isinstance(node, ast.Call): + forward_ref = typing.ForwardRef + binding = evaluate.eval_node(context.environment, node.func) + origin = None if binding is None else binding.get_object() + attribute = None + if binding is None and isinstance(node.func, ast.Attribute): + binding = evaluate.eval_node(context.environment, node.func.value) + attribute = node.func.attr + if ( + builtins._is_runtime_class( + origin, forward_ref, binding=binding, attribute=attribute + ) + and node.args + ): + yield from self._quoted_type_operands( + node.args[0], source, coordinates, context, name + ) + elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + for operand in (node.left, node.right): + yield from self._quoted_type_operands( + operand, source, coordinates, context, name + ) + elif isinstance(node, (ast.Tuple, ast.List)): + for operand in node.elts: + yield from self._quoted_type_operands( + operand, source, coordinates, context, name + ) + + def children_of(self, environment): + self._complete() + if isinstance(environment, BindingEnvironment): + return tuple(environment.children) + return tuple( + candidate + for candidate in self.environments + if _resolved(candidate.parent) is environment + ) + + def _capture(self, environment): + if isinstance(environment, TypeAliasEnvironment): + return environment.capture_parent + if isinstance(environment, AnnotationEnvironment): + if environment.generic_names: + capture = AnnotationEnvironment( + self, + environment.node, + environment.capture_parent, + environment.structural_reference, + environment.policy, + environment.generic_names, + environment.candidates, + ) + capture.opaque_names = environment.opaque_names + return self._environment(capture) + return environment.capture_parent + if ( + isinstance(environment, _GenericOwnerEnvironment) + and environment.kind == "Class" + ): + return _PropagationView(environment.parent) + if isinstance(environment, TypeParameterEnvironment): + return _PropagationView(environment) + if ( + isinstance(environment, _StructuralReference) + and environment.get_kind() == "Class" + ): + return self._capture(environment.entry) + return environment + + def _candidate_environments(self, environment): + result = [] + while environment is not None: + if environment not in result: + result.append(environment) + environment = environment.parent + if self.global_reference not in result: + result.append(self.global_reference) + return result + + def _environment(self, environment): + self.environments.append(environment) + if isinstance(environment.parent, BindingEnvironment): + environment.parent.children.append(environment) + return environment + + def _token(self, node, name, binding): + start = self._span(node)[0] + while self.source[start : start + 1] == "*": + start += 1 + self.token_bindings.append((start, start + len(name), binding)) + + def _generic_names(self, environment): + names = set() + while environment is not None: + if isinstance(environment, TypeParameterEnvironment): + names.update(environment.names) + environment = environment.parent + return names + + def _class_fallback(self, reference, name): + names, globals_, nonlocals = self._class_lexical_symbols(reference.node) + if name in globals_ or (name in names and name not in nonlocals): + return self.global_reference + return _PropagationView(reference.entry) + + def _class_lexical_symbols(self, node): + cached = self._class_symbols.get(id(node)) + if cached is not None: + return cached + names, globals_, nonlocals = set(), set(), set() + + def visit(expression): + if isinstance(expression, (ast.FunctionDef, ast.AsyncFunctionDef)): + names.add(expression.name) + headers = expression.decorator_list + expression.args.defaults + headers += [ + value for value in expression.args.kw_defaults if value is not None + ] + if self.policy.mode == "eager": + arguments = ( + expression.args.posonlyargs + + expression.args.args + + expression.args.kwonlyargs + ) + arguments += [ + arg + for arg in (expression.args.vararg, expression.args.kwarg) + if arg is not None + ] + headers += [ + arg.annotation + for arg in arguments + if arg.annotation is not None + ] + if expression.returns is not None: + headers.append(expression.returns) + for header in headers: + visit(header) + return + if isinstance(expression, ast.ClassDef): + names.add(expression.name) + for header in expression.decorator_list + expression.bases: + visit(header) + for keyword in expression.keywords: + visit(keyword.value) + return + if isinstance(expression, ast.Lambda): + for default in expression.args.defaults + [ + value for value in expression.args.kw_defaults if value is not None + ]: + visit(default) + return + if isinstance( + expression, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp) + ): + visit(expression.generators[0].iter) + return + if isinstance(expression, ast.Name) and isinstance( + expression.ctx, (ast.Store, ast.Del) + ): + names.add(expression.id) + elif isinstance(expression, ast.Global): + globals_.update(expression.names) + elif isinstance(expression, ast.Nonlocal): + nonlocals.update(expression.names) + elif isinstance(expression, (ast.Import, ast.ImportFrom)): + names.update( + alias.asname or alias.name.split(".")[0] + for alias in expression.names + ) + elif isinstance(expression, ast.ExceptHandler) and expression.name: + names.add(expression.name) + elif ( + isinstance(expression, (ast.MatchAs, ast.MatchStar)) and expression.name + ): + names.add(expression.name) + elif isinstance(expression, ast.MatchMapping) and expression.rest: + names.add(expression.rest) + for child in ast.iter_child_nodes(expression): + visit(child) + + for statement in node.body: + visit(statement) + result = names, globals_, nonlocals + self._class_symbols[id(node)] = result + return result + + def _class_binding_before(self, node, name, expression): + status = "absent" + for statement in node.body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + headers = list(statement.decorator_list) + list(statement.args.defaults) + headers.extend( + value for value in statement.args.kw_defaults if value is not None + ) + if self.policy.mode == "eager": + arguments = statement.args.posonlyargs + statement.args.args + arguments += ( + [statement.args.vararg] + if statement.args.vararg is not None + else [] + ) + arguments += statement.args.kwonlyargs + arguments += ( + [statement.args.kwarg] + if statement.args.kwarg is not None + else [] + ) + headers.extend( + arg.annotation + for arg in arguments + if arg.annotation is not None + ) + if statement.returns is not None: + headers.append(statement.returns) + for header in headers: + if expression is header: + if self._header_binding_status(header, name) != "absent": + return "ambiguous" + return status + if expression is not None and statement.lineno > expression.lineno: + break + header_status = self._header_binding_status(header, name) + if header_status != "absent": + status = header_status + if expression is not None and statement.lineno >= expression.lineno: + break + if statement.name == name: + status = "bound" + continue + if expression is not None and statement.lineno >= expression.lineno: + break + if isinstance(statement, ast.ClassDef): + if statement.name == name: + status = "bound" + continue + if _TYPE_ALIAS_TYPES and isinstance(statement, _TYPE_ALIAS_TYPES): + if statement.name.id == name: + status = "bound" + continue + if isinstance(statement, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + if isinstance(statement, ast.AnnAssign) and statement.value is None: + continue + targets = ( + statement.targets + if isinstance(statement, ast.Assign) + else [statement.target] + ) + if any( + isinstance(child, ast.Name) + and child.id == name + and isinstance(child.ctx, ast.Store) + for target in targets + for child in ast.walk(target) + ): + status = "bound" + continue + if isinstance(statement, (ast.Import, ast.ImportFrom)): + if any( + (alias.asname or alias.name.split(".")[0]) == name + for alias in statement.names + ): + status = "bound" + continue + if isinstance(statement, ast.Delete): + if any( + isinstance(child, ast.Name) and child.id == name + for child in ast.walk(statement) + ): + status = "absent" + continue + if any( + ( + isinstance(child, ast.Name) + and child.id == name + and isinstance(child.ctx, (ast.Store, ast.Del)) + ) + or ( + isinstance(child, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)) + and child.name == name + ) + or (isinstance(child, ast.MatchMapping) and child.rest == name) + for child in ast.walk(statement) + ): + status = "ambiguous" + return status + + def _header_binding_status(self, expression, name): + if isinstance(expression, ast.NamedExpr) and expression.target.id == name: + return "bound" + if isinstance(expression, ast.Lambda): + expressions = expression.args.defaults + [ + value for value in expression.args.kw_defaults if value is not None + ] + statuses = [ + self._header_binding_status(value, name) for value in expressions + ] + return next( + (status for status in reversed(statuses) if status != "absent"), + "absent", + ) + if any( + isinstance(child, ast.NamedExpr) and child.target.id == name + for child in ast.walk(expression) + ): + return "ambiguous" + return "absent" + + def _header_binding_owner(self, owner, name): + if owner.parent is None: + return owner + _, globals_, nonlocals = self._class_lexical_symbols(owner.node) + if name in globals_: + return self.global_reference + if name not in nonlocals: + return owner + outer = owner.parent + while outer is not None: + if outer.get_kind() == "Function": + names, outer_globals, outer_nonlocals = self._class_lexical_symbols( + outer.node + ) + args = outer.node.args + parameters = args.posonlyargs + args.args + args.kwonlyargs + parameters += [ + arg for arg in (args.vararg, args.kwarg) if arg is not None + ] + locals_ = names | {arg.arg for arg in parameters} + if ( + name in locals_ + and name not in outer_globals + and name not in outer_nonlocals + ): + return outer + outer = outer.parent + raise exceptions.RefactoringError( + "Cannot resolve the declared nonlocal source binding." + ) + + def _register_header_binding(self, owner, target, value): + declaring = self._header_binding_owner(owner, target.id) + declaring.header_slots.setdefault(target.id, []).append( + pynames.AssignmentValue(value, branch_context=self._branch_context) + ) + if declaring is not owner: + owner.redirected_headers[target.id] = declaring + self._token(target, target.id, (declaring, target.id)) + return declaring + + def _typeparams(self, node, entry, owner, depth, visit_expressions=True): + parameters = getattr(node, "type_params", ()) + if not parameters: + return entry + environment = self._environment( + TypeParameterEnvironment(self, node, entry, owner, "TypeParameters") + ) + for parameter in parameters: + name = parameter.name + binding = TypeParameterName(environment, name, parameter) + environment.names[name] = binding + self._record(parameter, environment, owner, "type-parameter-token", depth) + self._token(parameter, name, binding) + if visit_expressions: + for parameter in parameters: + for field in ("bound", "default_value"): + expression = getattr(parameter, field, None) + if expression is not None: + self._visit( + expression, + environment, + owner, + "type-parameter-expression", + depth + 1, + ) + return environment + + def _visit(self, node, environment, owner, role, depth, annotation=None): + previous = self._branch_context + if type(node).__name__ in ( + "For", + "AsyncFor", + "While", + "Try", + "TryStar", + "Match", + ): + self._branch_context = previous + ((node, None),) + try: + return self._visit_node(node, environment, owner, role, depth, annotation) + finally: + self._branch_context = previous + + def _visit_node(self, node, environment, owner, role, depth, annotation=None): + if node is None: + return + self._record(node, environment, owner, role, depth, annotation) + if isinstance(node, ast.If): + self._visit(node.test, environment, owner, role, depth + 1, annotation) + previous = self._branch_context + try: + for arm, statements in ((True, node.body), (False, node.orelse)): + self._branch_context = previous + ((node, arm),) + for statement in statements: + self._visit( + statement, environment, owner, role, depth + 1, annotation + ) + finally: + self._branch_context = previous + return + cls = node.__class__ + kind = _NODE_KINDS.get(cls, -1) + if kind < 0: + kind = _classify_node(node) + if kind == _KIND_FUNCTION: + self._function(node, environment, owner, depth) + return + if kind == _KIND_TYPEALIAS: + generic = self._typeparams( + node, environment, owner, depth + 1, visit_expressions=False + ) + alias_environment = self._environment( + TypeAliasEnvironment(self, node, environment, generic, owner) + ) + self._visit(node.name, environment, owner, "type-alias-name", depth + 1) + for parameter in node.type_params: + for field in ("bound", "default_value"): + expression = getattr(parameter, field, None) + if expression is not None: + self._visit( + expression, + alias_environment, + owner, + "type-parameter-expression", + depth + 2, + node, + ) + self._visit( + node.value, + alias_environment, + owner, + "type-alias-value", + depth + 1, + node, + ) + return + if kind == _KIND_CLASS: + class_owner = _StructuralReference(self, node, owner, environment) + self.owner_references.append(class_owner) + self._owners_by_node[id(node)] = class_owner + generic = self._typeparams(node, environment, class_owner, depth + 1) + for expression in node.decorator_list: + self._visit(expression, environment, owner, "decorator", depth + 1) + for expression in node.bases: + self._visit(expression, generic, class_owner, "class-base", depth + 1) + for keyword in node.keywords: + self._visit( + keyword.value, generic, class_owner, "class-keyword", depth + 1 + ) + body_env = class_owner + if self._generic_names(generic): + body_env = self._environment( + _GenericOwnerEnvironment(self, node, generic, class_owner, "Class") + ) + self._record(node, body_env, class_owner, "class-body", depth) + for statement in node.body: + self._visit(statement, body_env, class_owner, "class-body", depth + 1) + return + if kind == _KIND_LAMBDA: + self._lambda(node, environment, owner, depth, annotation) + return + if kind == _KIND_COMPREHENSION: + self._comprehension(node, environment, owner, depth, annotation) + return + if kind == _KIND_WALRUS: + target_env = environment + while isinstance(target_env, ComprehensionBindingScope): + target_env = target_env.parent + if isinstance(target_env, BindingEnvironment) and not isinstance( + target_env, AnnotationEnvironment + ): + if node.target.id not in target_env.names: + target_env.names[node.target.id] = SourceLocalName( + target_env, node.target.id, node.target + ) + self._token( + node.target, node.target.id, target_env.names[node.target.id] + ) + elif isinstance(target_env, _StructuralReference): + target_env = self._register_header_binding( + target_env, node.target, node.value + ) + elif ( + isinstance(target_env, AnnotationEnvironment) + and target_env.policy.mode == "eager" + ): + declaring = target_env.parent + while isinstance(declaring, BindingEnvironment): + declaring = declaring.parent + if isinstance(declaring, _StructuralReference): + target_env = self._register_header_binding( + declaring, node.target, node.value + ) + self._visit( + node.target, + target_env, + owner, + "named-expression-target", + depth + 1, + annotation, + ) + self._visit(node.value, environment, owner, role, depth + 1, annotation) + return + child_depth = depth + 1 + visit = self._visit + spec = _CHILD_SPECS.get(cls) + if spec is None: + spec = _child_spec(node) + for name, is_list in spec: + value = getattr(node, name) + if is_list: + for item in value: + if isinstance(item, ast.AST): + visit(item, environment, owner, role, child_depth, annotation) + elif isinstance(value, ast.AST): + visit(value, environment, owner, role, child_depth, annotation) + + def _function(self, node, entry, parent_owner, depth): + owner = _StructuralReference(self, node, parent_owner, entry) + self.owner_references.append(owner) + self._owners_by_node[id(node)] = owner + generic = self._typeparams(node, entry, owner, depth + 1) + args = node.args + parameters = args.posonlyargs + args.args + args.kwonlyargs + parameters += [arg for arg in (args.vararg, args.kwarg) if arg is not None] + for parameter in parameters: + self._record(parameter, owner, owner, "formal-token", depth + 1) + # Positional-only and keyword-only formals of ordinary functions + # resolve through the function scope's parameter protocol + # (get_parameters), which models every parameter category. + self._token(parameter, parameter.arg, (owner, parameter.arg)) + for expression in node.decorator_list: + self._visit(expression, entry, parent_owner, "decorator", depth + 1) + for expression in args.defaults + [ + value for value in args.kw_defaults if value is not None + ]: + self._visit(expression, entry, parent_owner, "function-default", depth + 1) + annotated_parameters = args.posonlyargs + args.args + annotated_parameters += [args.vararg] if args.vararg is not None else [] + annotated_parameters += args.kwonlyargs + annotated_parameters += [args.kwarg] if args.kwarg is not None else [] + annotations = [ + (arg.annotation, "parameter-annotation") for arg in annotated_parameters + ] + annotations.append((node.returns, "return-annotation")) + future_annotations = self.future_annotations + if future_annotations: + resolver = self.global_reference + own_parameters = getattr(node, "type_params", ()) + names = set(generic.names) if own_parameters else set() + candidates = ( + (generic, self.global_reference) + if own_parameters + else (self.global_reference,) + ) + else: + resolver = generic + names = self._generic_names(generic) + candidates = None + for expression, role in annotations: + if expression is None: + continue + annotation_env = self._environment( + AnnotationEnvironment( + self, expression, resolver, owner, self.policy, names, candidates + ) + ) + self._visit(expression, annotation_env, owner, role, depth + 2, node) + body_env = owner + if self._generic_names(generic): + body_env = self._environment( + _GenericOwnerEnvironment(self, node, generic, owner, "Function") + ) + previous = self._branch_context + self._branch_context = () + try: + self._record(node, body_env, owner, "function-body", depth) + if self._defer_bodies: + self._pending_bodies[id(node)] = (node, body_env, owner, depth) + else: + for statement in node.body: + self._visit(statement, body_env, owner, "function-body", depth + 1) + finally: + self._branch_context = previous + + def _lambda(self, node, entry, owner, depth, annotation): + environment = self._environment( + LambdaBindingScope(self, node, self._capture(entry), owner, "Lambda") + ) + callable_ = SourceLambdaFunction(self, node, entry, environment) + environment.pyobject = callable_ + self._record(node, environment, owner, "lambda-body", depth, annotation) + self.source_callables[id(node)] = callable_ + parameters = node.args.posonlyargs + node.args.args + node.args.kwonlyargs + parameters += [ + arg for arg in (node.args.vararg, node.args.kwarg) if arg is not None + ] + for number, parameter in enumerate(parameters): + binding = LambdaParameterName(callable_, parameter.arg, number, parameter) + environment.names[parameter.arg] = binding + self._record( + parameter, environment, owner, "lambda-formal", depth + 1, annotation + ) + self._token(parameter, parameter.arg, binding) + previous = self._branch_context + self._branch_context = () + try: + self._predeclare_walrus(node.body, environment) + finally: + self._branch_context = previous + for expression in node.args.defaults + [ + value for value in node.args.kw_defaults if value is not None + ]: + self._visit( + expression, entry, owner, "lambda-default", depth + 1, annotation + ) + self._branch_context = () + try: + self._visit( + node.body, environment, owner, "lambda-body", depth + 1, annotation + ) + finally: + self._branch_context = previous + + def _predeclare_walrus(self, node, environment): + if isinstance(node, ast.Lambda): + return + if isinstance(node, ast.NamedExpr): + if node.target.id not in environment.names: + environment.names[node.target.id] = SourceLocalName( + environment, node.target.id, node.target + ) + binding = environment.names[node.target.id] + if isinstance(binding, SourceLocalName): + binding.assignments.append( + pynames.AssignmentValue( + node.value, branch_context=self._branch_context + ) + ) + for child in ast.iter_child_nodes(node): + self._predeclare_walrus(child, environment) + + def _comprehension(self, node, entry, owner, depth, annotation): + environment = self._environment( + ComprehensionBindingScope( + self, node, self._capture(entry), owner, "Comprehension" + ) + ) + self.comprehensions[id(node)] = environment + self._record(node, environment, owner, "comprehension-inner", depth, annotation) + for generator in node.generators: + for child in ast.walk(generator.target): + if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Store): + if child.id not in environment.names: + environment.names[child.id] = SourceLocalName( + environment, child.id, child + ) + self._token(child, child.id, environment.names[child.id]) + for name, levels in nameanalyze.get_name_levels(generator.target): + environment.names[name].assignments.append( + pynames.AssignmentValue( + generator.iter, + levels, + ".__iter__().next()", + branch_context=self._branch_context, + ) + ) + for number, generator in enumerate(node.generators): + self._visit( + generator.iter, + entry if number == 0 else environment, + owner, + "comprehension-first-iter" if number == 0 else "comprehension-inner", + depth + 1, + annotation, + ) + self._visit( + generator.target, + environment, + owner, + "comprehension-store", + depth + 1, + annotation, + ) + for condition in generator.ifs: + self._visit( + condition, + environment, + owner, + "comprehension-inner", + depth + 1, + annotation, + ) + for field in ("elt", "key", "value"): + expression = getattr(node, field, None) + if expression is not None: + self._visit( + expression, + environment, + owner, + "comprehension-inner", + depth + 1, + annotation, + ) + + +class _GenericOwnerEnvironment(BindingEnvironment): + def lookup(self, name): + own = self.structural_scope.get_names().get(name) + return own if own is not None else self.parent._propagated_lookup(name) + + def _propagated_lookup(self, name): + if self.kind == "Class": + return self.parent._propagated_lookup(name) + return self.lookup(name) + + def get_names(self): + return self.structural_scope.get_names() diff --git a/rope/base/utils/__init__.py b/rope/base/utils/__init__.py index c96c96428..bfef93c91 100644 --- a/rope/base/utils/__init__.py +++ b/rope/base/utils/__init__.py @@ -1,13 +1,37 @@ +import os import sys import warnings +from typing import Callable, Concatenate, ParamSpec, TypeVar +_Self = TypeVar("_Self") +_Params = ParamSpec("_Params") +_Result = TypeVar("_Result") -def saveit(func): + +def same_file_path(first, second): + """Tell whether two filesystem paths name the same existing file. + + Runtime ``__file__`` values and project resource paths may spell one + file differently (letter case on Windows: ``lib`` vs ``Lib``, links), so + identity is the filesystem's. A path that does not name an existing + file (a frozen or zipped module) is not that source file. + """ + if not isinstance(first, str) or not isinstance(second, str): + return False + try: + return os.path.samefile(first, second) + except (OSError, ValueError): + return False + + +def saveit( + func: Callable[Concatenate[_Self, _Params], _Result], +) -> Callable[Concatenate[_Self, _Params], _Result]: """A decorator that caches the return value of a function""" name = "_" + func.__name__ - def _wrapper(self, *args, **kwds): + def _wrapper(self: _Self, *args: _Params.args, **kwds: _Params.kwargs) -> _Result: if not hasattr(self, name): setattr(self, name, func(self, *args, **kwds)) return getattr(self, name) @@ -74,7 +98,7 @@ def cached(size): def decorator(func): cached_func = _Cached(func, size) - return lambda *a, **kw: cached_func(*a, **kw) + return cached_func return decorator @@ -96,6 +120,23 @@ def __call__(self, *args, **kwds): del self.cache[0] return result + def __get__(self, instance, owner=None): + # Decorating a method needs instance binding; otherwise the method's + # ``self`` would be consumed as the first positional cache argument + # and the real signature would lose a parameter. + if instance is None: + return self + return _CachedBound(self, instance) + + +class _CachedBound: + def __init__(self, cached, instance): + self.cached = cached + self.instance = instance + + def __call__(self, *args, **kwds): + return self.cached(self.instance, *args, **kwds) + def resolve(str_or_obj): """Returns object from string""" diff --git a/rope/base/worder.py b/rope/base/worder.py index 97ea8d405..c787d4558 100644 --- a/rope/base/worder.py +++ b/rope/base/worder.py @@ -97,8 +97,8 @@ def is_assigned_here(self, offset): def is_a_function_being_called(self, offset): return self.code_finder.is_a_function_being_called(offset) - def get_word_parens_range(self, offset): - return self.code_finder.get_word_parens_range(offset) + def get_word_parens_range(self, offset, opening="(", closing=")"): + return self.code_finder.get_word_parens_range(offset, opening, closing) def is_name_assigned_in_class_body(self, offset): return self.code_finder.is_name_assigned_in_class_body(offset) @@ -489,6 +489,12 @@ def get_assignment_type(self, offset): if double not in ("==", "<=", ">=", "!="): for op in [single, double, triple]: if op.endswith("="): + for end in range(next_char, next_char + len(op) - 1): + if self.code[end] in ")]": + # A subscript index is read, unlike an unpacking target. + parens_start = self._find_parens_start(end) + if self._find_primary_start(end) < parens_start: + return None return op def get_primary_range(self, offset): diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index 7ae8fb718..f513fbd37 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -4,6 +4,7 @@ import contextlib import json +import multiprocessing import re import secrets import sqlite3 @@ -11,11 +12,11 @@ import warnings from collections import OrderedDict from concurrent.futures import Future, ProcessPoolExecutor, as_completed -from datetime import datetime +from datetime import datetime, timezone from hashlib import sha256 from itertools import chain from pathlib import Path -from threading import local +from threading import Lock, local from typing import ( TYPE_CHECKING, Generator, @@ -44,7 +45,6 @@ from rope.contrib.autoimport.parse import get_names from rope.contrib.autoimport.utils import ( get_files, - get_modname_from_path, get_package_tuple, sort_and_deduplicate, sort_and_deduplicate_tuple, @@ -55,11 +55,23 @@ from collections.abc import Collection +def _process_pool() -> ProcessPoolExecutor: + """Return a pool whose workers never come from forking this process. + + Forking a process that has threads (an editor, a language server or a + test runner usually do) can deadlock the child. ``forkserver`` (the + POSIX default since Python 3.14) or ``spawn`` start workers cleanly. + """ + methods = multiprocessing.get_all_start_methods() + method = "forkserver" if "forkserver" in methods else "spawn" + return ProcessPoolExecutor(mp_context=multiprocessing.get_context(method)) + + def get_future_names( packages: List[Package], underlined: bool, job_set: taskhandle.BaseJobSet ) -> Generator[Future[Collection[Name]], None, None]: """Get all names as futures.""" - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for package in packages: for module in get_files(package, underlined): job_set.started_job(module.modname) @@ -142,6 +154,10 @@ def __init__( "`AutoImport(memory=True)` explicitly.", DeprecationWarning, ) + self._closed = False + self._connections: Set[sqlite3.Connection] = set() + self._connections_lock = Lock() + self._observer: Optional[resourceobserver.ResourceObserver] = None self.thread_local = local() self.connection = self.create_database_connection( project=project, @@ -149,10 +165,10 @@ def __init__( ) self._setup_db() if observe: - observer = resourceobserver.ResourceObserver( + self._observer = resourceobserver.ResourceObserver( changed=self._changed, moved=self._moved, removed=self._removed ) - project.add_observer(observer) + project.add_observer(self._observer) @classmethod def create_database_connection( @@ -186,10 +202,25 @@ def calculate_project_hash(data: str) -> str: else: project_hash = calculate_project_hash(project.ropefolder.real_path) return sqlite3.connect( - f"file:rope-{project_hash}:?mode=memory&cache=shared", uri=True + f"file:rope-{project_hash}:?mode=memory&cache=shared", + uri=True, + check_same_thread=False, ) else: - return sqlite3.connect(project.ropefolder.pathlib / "autoimport.db") + return sqlite3.connect( + project.ropefolder.pathlib / "autoimport.db", + check_same_thread=False, + ) + + def _register_connection(self, conn: sqlite3.Connection) -> None: + with self._connections_lock: + if self._closed: + with contextlib.suppress( + sqlite3.ProgrammingError, sqlite3.OperationalError + ): + conn.close() + raise exceptions.RopeError("AutoImport instance has been closed") + self._connections.add(conn) @property def connection(self) -> sqlite3.Connection: @@ -198,15 +229,26 @@ def connection(self) -> sqlite3.Connection: This makes sure AutoImport can be shared across threads. """ + if self._closed: + raise exceptions.RopeError("AutoImport instance has been closed") if not hasattr(self.thread_local, "connection"): - self.thread_local.connection = self.create_database_connection( + conn = self.create_database_connection( project=self.project, memory=self.memory, ) + self._register_connection(conn) + self.thread_local.connection = conn return self.thread_local.connection @connection.setter def connection(self, value: sqlite3.Connection): + if self._closed: + raise exceptions.RopeError("AutoImport instance has been closed") + old_conn = getattr(self.thread_local, "connection", None) + if old_conn is not None and old_conn is not value: + with self._connections_lock: + self._connections.discard(old_conn) + self._register_connection(value) self.thread_local.connection = value def _setup_db(self): @@ -390,7 +432,7 @@ def generate_cache( models.Package.delete_by_package_name, (self.project_package.name,) ) futures = [] - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for file in resources: job_set.started_job(f"Working on {file.path}") module = self._resource_to_module(file, underlined) @@ -457,10 +499,56 @@ def update_module(self, module: str): self._del_if_exist(module) self.generate_modules_cache([module]) + def close_thread_connection(self): + """Close the SQLite connection for the current thread.""" + conn = getattr(self.thread_local, "connection", None) + if conn is not None: + with self._connections_lock: + self._connections.discard(conn) + with contextlib.suppress(AttributeError): + del self.thread_local.connection + with contextlib.suppress( + sqlite3.ProgrammingError, sqlite3.OperationalError + ): + conn.commit() + with contextlib.suppress( + sqlite3.ProgrammingError, sqlite3.OperationalError + ): + conn.close() + def close(self): """Close the autoimport database.""" - self.connection.commit() - self.connection.close() + with self._connections_lock: + if self._closed: + return + self._closed = True + connections = list(self._connections) + self._connections.clear() + + if self._observer is not None: + with contextlib.suppress(Exception): + self.project.remove_observer(self._observer) + self._observer = None + + for conn in connections: + with contextlib.suppress( + sqlite3.ProgrammingError, sqlite3.OperationalError + ): + conn.commit() + with contextlib.suppress( + sqlite3.ProgrammingError, sqlite3.OperationalError + ): + conn.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def __del__(self): + with contextlib.suppress(Exception): + self.close() def get_name_locations(self, name): """Return a list of ``(resource, lineno)`` tuples.""" @@ -503,7 +591,7 @@ def clear_cache(self): data = ( versioning.calculate_version_hash(self.project), json.dumps(versioning.get_version_hash_data(self.project)), - datetime.utcnow().isoformat(), + datetime.now(timezone.utc).isoformat(), ) assert models.Metadata.columns == [ "version_hash", @@ -653,12 +741,13 @@ def _resource_to_module( assert self.project_package.path underlined = underlined if underlined else self.underlined resource_path: Path = resource.pathlib - # The project doesn't need its name added to the path, - # since the standard python file layout accounts for that - # so we set add_package_name to False - resource_modname: str = get_modname_from_path( - resource_path, self.project_package.path, add_package_name=False - ) + # Resolve the module name by walking up the package hierarchy from the + # resource (stopping at the first directory without an ``__init__.py``), + # rather than deriving it from the resource's path relative to the + # project root. The latter incorrectly includes intermediate, + # non-package directories in the module name. + # See https://github.com/python-rope/rope/issues/823 + resource_modname: str = libutils.modname(resource) return ModuleFile( resource_path, resource_modname, diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 324ea3173..b96cd5015 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -5,6 +5,7 @@ from collections import OrderedDict from typing import Generator, List, Optional, Tuple +from rope.base import libutils from rope.base.project import Project from .defs import ModuleCompiled, ModuleFile, ModuleInfo, Package, PackageType, Source @@ -49,10 +50,10 @@ def get_package_source( """Detect the source of a given package. Rudimentary implementation.""" if name in sys.builtin_module_names: return Source.BUILTIN - if project is not None and package.resolve().is_relative_to( - project.root.pathlib.resolve() - ): - return Source.PROJECT + if project is not None: + relative = libutils.path_relative_to_project_root(project, str(package)) + if relative is not None and not project.is_ignored(project.get_file(relative)): + return Source.PROJECT if "site-packages" in package.parts: return Source.SITE_PACKAGE if sys.version_info < (3, 10, 0): @@ -71,6 +72,14 @@ def get_modname_from_path( package_name: str = package_path.stem rel_path_parts = modpath.relative_to(package_path).parts modname = "" + try: + site_packages_index = rel_path_parts.index("site-packages") + raise RuntimeError("No site-packages allowed here.", modpath) + except ValueError: + pass + else: + # If path includes "site-packages", we're interested in part after this. + rel_path_parts = rel_path_parts[site_packages_index + 1 :] if len(rel_path_parts) > 0: for part in rel_path_parts[:-1]: modname += part @@ -122,6 +131,9 @@ def get_files( else: assert package.path for file in package.path.glob("**/*.py"): + # Nested installation roots are discovered as separate packages. + if "site-packages" in file.relative_to(package.path).parts: + continue if file.name == "__init__.py": yield ModuleFile( file, diff --git a/rope/contrib/codeassist.py b/rope/contrib/codeassist.py index 32547891e..27516ba76 100644 --- a/rope/contrib/codeassist.py +++ b/rope/contrib/codeassist.py @@ -1,9 +1,13 @@ +import io import keyword import sys +import tokenize import warnings from rope.base import ( + ast, builtins, + codeanalyze, evaluate, exceptions, libutils, @@ -335,6 +339,13 @@ def get_default(self): Returns None if there is no default value for this param. """ + if isinstance(self._function, pyobjects.PyFunction): + default = self._function.get_parameter_defaults().get(self.argname) + if default is None: + return None + return ast.get_source_segment( + self._function.get_module().source_code, default + ) definfo = functionutils.DefinitionInfo.read(self._function) for arg, default in definfo.args_with_defaults: if self.argname == arg: @@ -471,6 +482,13 @@ def _find_module(self, pymodule, module_name): return pyname.get_object() def _is_defined_after(self, scope, pyname, lineno): + binding_scope = getattr(pyname, "binding_scope", None) + if binding_scope is not None and binding_scope.get_kind() in { + "Lambda", + "Comprehension", + "TypeParameters", + }: + return False location = pyname.get_definition_location() if location is not None and location[1] is not None: if ( @@ -492,6 +510,25 @@ def _code_completions(self): start = fixsyntax._logical_start(lines, lineno) indents = fixsyntax._get_line_indents(lines[start - 1]) inner_scope = module_scope.get_inner_scope_for_line(start, indents) + transferred = fixer.commenter.transferred_offset(self.offset) + if code.startswith(self.code[: self.offset]): + context = pymodule.get_source_binding_index().context_at( + max(0, transferred if self.starting else transferred - 1) + ) + if context.role in { + "parameter-annotation", + "return-annotation", + "function-default", + "lambda-formal", + "lambda-default", + "lambda-body", + "comprehension-first-iter", + "comprehension-inner", + "comprehension-store", + "type-parameter-token", + "type-parameter-expression", + }: + inner_scope = context.environment if self.word_finder.is_a_name_after_from_import(self.offset): return self._from_import_completions(pymodule) if self.expression.strip() != "": @@ -526,7 +563,7 @@ def _keyword_parameters(self, pymodule, scope): pyobject = pyobject["__call__"].get_object() if isinstance(pyobject, pyobjects.AbstractFunction): param_names = [] - param_names.extend(pyobject.get_param_names(special_args=False)) + param_names.extend(pyobject.get_keyword_param_names()) result = {} for name in param_names: if name.startswith(self.starting): @@ -609,8 +646,14 @@ def get_calltip(self, pyobject, ignore_unknown=False, remove_self=False): if ignore_unknown and not isinstance(pyobject, pyobjects.PyFunction): return if isinstance(pyobject, pyobjects.AbstractFunction): - result = self._get_function_signature(pyobject, add_module=True) - if remove_self and self._is_method(pyobject): + result = self._get_function_signature( + pyobject, add_module=True, remove_self=remove_self + ) + if ( + remove_self + and self._is_method(pyobject) + and not self._has_parameter_kinds(pyobject) + ): return result.replace("(self)", "()").replace("(self, ", "(") return result @@ -663,17 +706,90 @@ def _get_super_methods(self, pyclass, name): result.extend(self._get_super_methods(super_class, name)) return result - def _get_function_signature(self, pyfunction, add_module=False): + def _get_function_signature(self, pyfunction, add_module=False, remove_self=False): location = self._location(pyfunction, add_module) if isinstance(pyfunction, pyobjects.PyFunction): - info = functionutils.DefinitionInfo.read(pyfunction) - return location + info.to_string() + if self._has_parameter_kinds(pyfunction): + signature = functionutils._get_function_signature(pyfunction) + if signature.endswith(":"): + signature = signature[:-1] + if remove_self and self._is_method(pyfunction): + signature = self._remove_self(pyfunction, signature) + return location + signature + return location + functionutils.DefinitionInfo.read(pyfunction).to_string() else: return "{}({})".format( location + pyfunction.get_name(), ", ".join(pyfunction.get_param_names()), ) + @staticmethod + def _has_parameter_kinds(pyfunction): + args = pyfunction.arguments + return bool(args.posonlyargs or args.kwonlyargs) + + @staticmethod + def _remove_self(pyfunction, signature): + args = pyfunction.arguments + positional = args.posonlyargs + args.args + if not positional or positional[0].arg != "self": + return signature + receiver = positional[0] + defaults = pyfunction.get_parameter_defaults() + last = defaults.get("self", receiver) + pymodule = pyfunction.get_module() + lines = pymodule.lines + + def offset(lineno, column): + prefix = lines.get_line(lineno).encode("utf-8")[:column] + return lines.get_line_start(lineno) + len(prefix.decode("utf-8")) + + origin = pymodule.source_code.index( + signature, lines.get_line_start(pyfunction.get_ast().lineno) + ) + start = offset(receiver.lineno, receiver.col_offset) - origin + end = offset(last.end_lineno, last.end_col_offset) - origin + signature_lines = codeanalyze.SourceLinesAdapter(signature) + + def token_offset(token): + return signature_lines.get_line_start(token.start[0]) + token.start[1] + + tokens = [ + token + for token in tokenize.generate_tokens(io.StringIO(signature).readline) + if token.type + not in (tokenize.COMMENT, tokenize.NL, tokenize.NEWLINE, tokenize.ENDMARKER) + and token_offset(token) >= start + ] + depth = 0 + boundary = None + for index, token in enumerate(tokens): + if token.string in ("(", "[", "{"): + depth += 1 + elif token.string in (")", "]", "}"): + if depth: + depth -= 1 + else: + boundary = index + break + elif token.string == "," and depth == 0 and token_offset(token) >= end: + boundary = index + break + if boundary is None: + return signature + index = boundary + (tokens[boundary].string == ",") + if ( + len(args.posonlyargs) == 1 + and index < len(tokens) + and tokens[index].string == "/" + ): + index += 1 + if index < len(tokens) and tokens[index].string == ",": + index += 1 + if index < len(tokens): + end = token_offset(tokens[index]) + return signature[:start] + signature[end:] + def _location(self, pyobject, add_module=False): location = [] parent = pyobject.parent diff --git a/rope/contrib/finderrors.py b/rope/contrib/finderrors.py index 32ee31f98..ac5dd6d82 100644 --- a/rope/contrib/finderrors.py +++ b/rope/contrib/finderrors.py @@ -47,7 +47,9 @@ def __init__(self, pymodule): def _Name(self, node): if isinstance(node.ctx, (ast.Store, ast.Param)): return - scope = self.scope.get_inner_scope_for_line(node.lineno) + scope = ( + self.pymodule.get_source_binding_index().context_for_node(node).environment + ) pyname = scope.lookup(node.id) if pyname is None: self._add_error(node, "Unresolved variable") @@ -56,7 +58,11 @@ def _Name(self, node): def _Attribute(self, node): if not isinstance(node.ctx, ast.Store): - scope = self.scope.get_inner_scope_for_line(node.lineno) + scope = ( + self.pymodule.get_source_binding_index() + .context_for_node(node) + .environment + ) pyname = evaluate.eval_node(scope, node.value) if pyname is not None and pyname.get_object() != pyobjects.get_unknown(): if node.attr not in pyname.get_object(): @@ -73,6 +79,15 @@ def _add_error(self, node, msg): self.errors.append(error) def _is_defined_after(self, scope, pyname, lineno): + binding_scope = getattr(pyname, "binding_scope", None) + if not getattr(pyname, "is_refactoring_identity_safe", True): + return False + if binding_scope is not None and binding_scope.get_kind() in { + "Lambda", + "Comprehension", + "TypeParameters", + }: + return False location = pyname.get_definition_location() if location is not None and location[1] is not None: if ( diff --git a/rope/py.typed b/rope/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/rope/refactor/change_signature.py b/rope/refactor/change_signature.py index c6671b084..5857b56f3 100644 --- a/rope/refactor/change_signature.py +++ b/rope/refactor/change_signature.py @@ -1,9 +1,107 @@ +import contextlib import copy +import importlib +import sys +from functools import partial +from keyword import iskeyword import rope.base.exceptions -from rope.base import codeanalyze, evaluate, pyobjects, taskhandle, utils, worder +from rope.base import ( + ast, + builtins, + classfunctions, + codeanalyze, + evaluate, + pynames, + pyobjects, + simplify, + taskhandle, + utils, + worder, +) from rope.base.change import ChangeContents, ChangeSet -from rope.refactor import functionutils, occurrences +from rope.base.oi import soa +from rope.refactor import functionutils, occurrences, sourceutils + + +def _fstring_quotes(pymodule): + """Return ``(start, end, quote)`` for every f-string of `pymodule`.""" + index = pymodule.get_source_binding_index() + result = [] + for candidate in ast.walk(pymodule.get_ast()): + if isinstance(candidate, ast.JoinedStr): + first, last = index._span(candidate) + text = pymodule.source_code[first:last].lstrip("rRfFbBuU") + quote = text[:3] if text[:3] in ("'''", '"""') else text[:1] + result.append((first, last, quote)) + return result + + +def _enclosing_fstring_quote(fstrings, start, end): + """Return the quote of the innermost f-string around ``[start, end)``.""" + enclosing = [ + (last - first, quote) + for first, last, quote in fstrings + if first <= start and end <= last + ] + return min(enclosing)[1] if enclosing else None + + +def _replacement_field_text(expression, quote): + """Render `expression` for a replacement field before PEP 701. + + Before Python 3.12 an f-string replacement field cannot contain its + enclosing quote or a backslash. String literals anywhere in the + expression are written with the other quote character; an expression + that still cannot be written there is refused. + """ + text = ast.unparse(expression) + if quote is None or sys.version_info >= (3, 12): + return text + if quote not in text and "\\" not in text: + return text + other = '"' if quote[0] == "'" else "'" + literals = {} + + class _Requote(ast.NodeTransformer): + def visit_JoinedStr(self, node): + return node + + def visit_Constant(self, node): + value = node.value + if ( + not isinstance(value, str) + or other in value + or "\\" in value + or not value.isprintable() + ): + return node + name = "__rope_literal_%d__" % len(literals) + literals[name] = other + value + other + return ast.copy_location(ast.Name(id=name, ctx=ast.Load()), node) + + if "__rope_literal_" not in text: + text = ast.unparse(_Requote().visit(copy.deepcopy(expression))) + for name, literal in literals.items(): + text = text.replace(name, literal) + if quote in text or "\\" in text: + raise rope.base.exceptions.RefactoringError( + "Keyword value cannot be written inside this f-string before Python 3.12" + ) + return text + + +def _keyword_expression(source): + if not isinstance(source, str) or not source.strip(): + raise rope.base.exceptions.RefactoringError( + "Keyword addition requires an explicit Python expression" + ) + try: + return ast.parse(source.strip(), mode="eval").body + except SyntaxError as error: + raise rope.base.exceptions.RefactoringError( + "Keyword addition requires one valid Python expression" + ) from error class ChangeSignature: @@ -39,7 +137,10 @@ def _set_name_and_pyname(self): and isinstance(pyobject.parent, pyobjects.PyClass) ): pyclass = pyobject.parent - self.others = (pyclass.get_name(), pyclass.parent[pyclass.get_name()]) + self.others = ( + pyclass.get_name(), + pyclass.parent.get_scope()[pyclass.get_name()], + ) def _change_calls( self, @@ -58,11 +159,28 @@ def _change_calls( self.pyname, instance=self.primary, in_hierarchy=in_hierarchy and self.is_method(), + unsure=( + call_changer.refuse_unsure + if isinstance(call_changer, _KeywordAdditions) and call_changer.required + else None + ), ) if self.others: name, pyname = self.others constructor_finder = occurrences.create_finder( - self.project, name, pyname, only_calls=True + self.project, + name, + pyname, + only_calls=not ( + isinstance(call_changer, _KeywordAdditions) + and call_changer.required + ), + unsure=( + call_changer.refuse_unsure + if isinstance(call_changer, _KeywordAdditions) + and call_changer.required + else None + ), ) finder = _MultipleFinders([finder, constructor_finder]) for file in resources: @@ -74,6 +192,18 @@ def _change_calls( if changed_file is not None: changes.add_change(ChangeContents(file, changed_file)) job_set.finished_job() + if ( + isinstance(call_changer, _KeywordAdditions) + and call_changer.required + and changes.changes + ): + call_changer.resources = call_changer.resources | frozenset( + change.resource for change in changes.changes + ) + for file in sorted( + call_changer.resources, key=lambda resource: resource.path + ): + call_changer.validate_consumers(self.project.get_pymodule(file)) return changes def get_args(self): @@ -98,43 +228,48 @@ def _definfo(self): @utils.deprecated() def normalize(self): + functionutils._check_signature_parameters(self.pyname.get_object()) changer = _FunctionChangers( - self.pyname.get_object(), self.get_definition_info(), [ArgumentNormalizer()] + self.pyname.get_object(), self._definfo(), [ArgumentNormalizer()] ) return self._change_calls(changer) @utils.deprecated() def remove(self, index): + functionutils._check_signature_parameters(self.pyname.get_object()) changer = _FunctionChangers( self.pyname.get_object(), - self.get_definition_info(), + self._definfo(), [ArgumentRemover(index)], ) return self._change_calls(changer) @utils.deprecated() def add(self, index, name, default=None, value=None): + functionutils._check_signature_parameters(self.pyname.get_object()) changer = _FunctionChangers( self.pyname.get_object(), - self.get_definition_info(), + self._definfo(), [ArgumentAdder(index, name, default, value)], ) return self._change_calls(changer) @utils.deprecated() def inline_default(self, index): + functionutils._check_signature_parameters(self.pyname.get_object()) changer = _FunctionChangers( self.pyname.get_object(), - self.get_definition_info(), + self._definfo(), [ArgumentDefaultInliner(index)], ) return self._change_calls(changer) @utils.deprecated() def reorder(self, new_ordering): + functionutils._check_signature_parameters(self.pyname.get_object()) changer = _FunctionChangers( self.pyname.get_object(), - self.get_definition_info(), + self._definfo(), [ArgumentReorderer(new_ordering)], ) return self._change_calls(changer) @@ -156,6 +291,29 @@ def get_changes( in the project are searched. """ + if any( + isinstance(changer, ArgumentAdder) and changer.keyword_only + for changer in changers + ): + if not all( + isinstance(changer, ArgumentAdder) and changer.keyword_only + for changer in changers + ): + raise rope.base.exceptions.RefactoringError( + "Keyword-only additions cannot be mixed with legacy signature changes" + ) + if resources is None: + resources = self.project.get_python_files() + addition = _KeywordAdditions( + changers, self.pyname, self.name, self.others, resources + ) + return self._change_calls( + addition, + in_hierarchy, + resources, + task_handle, + ) + functionutils._check_signature_parameters(self.pyname.get_object()) function_changer = _FunctionChangers( self.pyname.get_object(), self._definfo(), changers ) @@ -181,7 +339,7 @@ def _get_changed_definition_infos(self): return result def change_definition(self, call): - return self.changed_definition_infos[-1].to_string() + return self.changed_definition_infos[-1].to_string(annotated=True) def change_call(self, primary, pyname, call): call_info = functionutils.CallInfo.read( @@ -240,11 +398,36 @@ def change_argument_mapping(self, definition_info, mapping): class ArgumentAdder(_ArgumentChanger): - def __init__(self, index, name, default=None, value=None): + def __init__( + self, + index, + name, + default=None, + value=None, + *, + keyword_only=False, + annotation=None, + ): self.index = index self.name = name self.default = default self.value = value + self.keyword_only = keyword_only + self.annotation = annotation + if keyword_only and ( + (default is None and value is None) + or not name.isidentifier() + or iskeyword(name) + ): + raise rope.base.exceptions.RefactoringError( + "Keyword addition needs an identifier and a call value or default" + ) + if keyword_only: + for expression in (annotation, default): + if expression is not None: + _keyword_expression(expression) + if value is not None and not callable(value): + _keyword_expression(value) def change_definition_info(self, definition_info): for pair in definition_info.args_with_defaults: @@ -313,6 +496,606 @@ def change_definition_info(self, definition_info): definition_info.args_with_defaults = new_args +class _KeywordAdditions: + """Insert required keywords using the native parse and original source bytes.""" + + def __init__(self, additions, pyname, name, constructor=None, resources=()): + self.additions = additions + self.required = any(addition.default is None for addition in additions) + self.resources = ( + self._source_component(pyname.get_object().get_module(), resources) + if self.required + else frozenset(resources) + ) + self.reflective_targets = ((name, pyname),) + ( + (constructor,) if constructor is not None else () + ) + if constructor is not None: + owner = constructor[1].get_object() + self._check_decorators(owner, owner.get_module().get_source_binding_index()) + native_new = ( + builtins.builtins["object"].get_object().get_attributes()["__new__"] + ) + custom_new = owner.get_attributes().get("__new__") + if ( + owner.get_metaclass() is not builtins.builtins["type"].get_object() + or any( + isinstance(base, pyobjects.UnresolvedBase) + for base in owner.get_mro() + ) + or custom_new is not None + and not occurrences.same_pyname(custom_new, native_new) + ): + raise rope.base.exceptions.RefactoringError( + "Constructor allocation has no proven native keyword behavior" + ) + if len({addition.name for addition in additions}) != len(additions): + raise rope.base.exceptions.RefactoringError( + "Adding duplicate keyword parameters" + ) + + @staticmethod + def refuse_unsure(occurrence): + raise rope.base.exceptions.RefactoringError( + "Unresolved signature consumer at %s:%s" + % (occurrence.resource.path, occurrence.lineno) + ) + + @staticmethod + def _source_component(module, resources): + resources = frozenset(resources) + root = module.get_resource() + if root not in resources: + raise rope.base.exceptions.RefactoringError( + "Signature owner is outside the analyzed source boundary" + ) + project = module.pycore.project + graph = {resource: set() for resource in resources} + for resource in resources: + current = project.get_pymodule(resource) + for call in ast.walk(current.get_ast()): + if not isinstance(call, ast.Call): + continue + scope = current.get_scope().get_inner_scope_for_line(call.lineno) + binding = evaluate.eval_node(scope, call.func) + function = binding.get_object() if binding is not None else None + if not ( + getattr(function, "builtin", None) is __import__ + or builtins._is_runtime_form(function, importlib.import_module) + ): + continue + if not ( + call.args + and isinstance(call.args[0], ast.Constant) + and isinstance(call.args[0].value, str) + and not call.args[0].value.startswith(".") + ): + raise rope.base.exceptions.RefactoringError( + f"{resource.path}:{call.lineno}: Unresolved dynamic import boundary" + ) + imported = project.find_module(call.args[0].value, resource.parent) + if ( + imported is not None + and imported.is_folder() + and imported.has_child("__init__.py") + ): + imported = imported.get_child("__init__.py") + if imported in graph: + graph[resource].add(imported) + graph[imported].add(resource) + for declaration in current.get_source_binding_index().import_layouts(): + if declaration.module_name is None: + names = [name for name, _, _ in declaration.aliases] + else: + provider = declaration.module_name + names = [provider] + [ + (provider + "." if provider else "") + name + for name, _, _ in declaration.aliases + if name != "*" + ] + for name in names: + imported = ( + project.find_relative_module( + name, resource.parent, declaration.level + ) + if declaration.level + else project.find_module(name, resource.parent) + ) + if ( + imported is not None + and imported.is_folder() + and imported.has_child("__init__.py") + ): + imported = imported.get_child("__init__.py") + if imported in graph: + graph[resource].add(imported) + graph[imported].add(resource) + pending = [root] + reached = set() + while pending: + current = pending.pop() + if current not in reached: + reached.add(current) + pending.extend(graph[current] - reached) + return frozenset(reached) + + def validate_consumers(self, pymodule): + soa.analyze_module( + pymodule.pycore, + pymodule, + lambda owner: True, + lambda owner: True, + sum( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + for resource in self.resources + for node in ast.walk( + pymodule.pycore.resource_to_pyobject(resource).get_ast() + ) + ), + visitor_factory=partial(_SignatureEffects, proof=self), + ) + + def changes(self, pymodule, occurrences): + source = pymodule.source_code + index = pymodule.get_source_binding_index() + calls = { + index._span(node.func): node + for node in ast.walk(pymodule.get_ast()) + if isinstance(node, ast.Call) + } + words = worder.Worder(source) + real = simplify.real_code(source) + fstrings = None + for occurrence in occurrences: + start, end = occurrence.get_primary_range() + node = calls.get((start, end)) + if occurrence.is_in_import_statement(): + if self.required and any( + alias is not None + and alias != name.rsplit(".", 1)[-1] + and first <= occurrence.offset < last + for layout in index.import_layouts() + for name, alias, (first, last) in layout.aliases + ): + raise rope.base.exceptions.RefactoringError( + "Signature consumer escapes through an import alias" + ) + continue + generator = False + if occurrence.is_defined(): + function = occurrence.get_pyname().get_object() + self._check_decorators(function, index) + if isinstance(function, pyobjects.PyClass): + continue + declaration = function.get_ast() + arguments = declaration.args + parameter_start = max( + ( + end, + *( + index._span(item)[1] + for item in getattr(declaration, "type_params", ()) + ), + ) + ) + begin, finish = words.get_word_parens_range(parameter_start - 1) + insertion = finish - 1 + if arguments.kwarg is not None: + insertion = real.rfind("**", begin, index._span(arguments.kwarg)[0]) + if insertion < begin: + raise rope.base.exceptions.RefactoringError( + "Cannot locate native keyword parameter" + ) + existing = { + argument.arg: argument + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + *((arguments.vararg,) if arguments.vararg else ()), + *((arguments.kwarg,) if arguments.kwarg else ()), + ) + } + added = [] + for addition in self.additions: + argument = existing.get(addition.name) + if argument is not None: + annotation = ( + ast.dump(argument.annotation) + if argument.annotation + else None + ) + expected_annotation = ( + ast.dump(_keyword_expression(addition.annotation)) + if addition.annotation is not None + else None + ) + if ( + argument not in arguments.kwonlyargs + or annotation != expected_annotation + ): + raise rope.base.exceptions.RefactoringError( + "Conflicting existing keyword parameter: " + + addition.name + ) + default = arguments.kw_defaults[ + arguments.kwonlyargs.index(argument) + ] + actual = ast.dump(default) if default is not None else None + expected_default = ( + ast.dump(_keyword_expression(addition.default)) + if addition.default is not None + else None + ) + if actual != expected_default: + raise rope.base.exceptions.RefactoringError( + "Conflicting existing keyword default: " + addition.name + ) + continue + added.append( + addition.name + + ( + ": " + ast.unparse(_keyword_expression(addition.annotation)) + if addition.annotation is not None + else "" + ) + + ( + " = " + ast.unparse(_keyword_expression(addition.default)) + if addition.default is not None + else "" + ) + ) + if not added: + continue + if arguments.vararg is None and not arguments.kwonlyargs: + added.insert(0, "*") + elif node is not None or occurrence.is_called(): + selected = [ + ( + addition, + ( + addition.value(pymodule, occurrence) + if callable(addition.value) + else addition.value + ), + ) + for addition in self.additions + ] + selected = [ + (addition, requested) + for addition, requested in selected + if requested is not None or addition.default is None + ] + if not selected: + continue + if node is None or any( + keyword.arg is None for keyword in node.keywords + ): + raise rope.base.exceptions.RefactoringError( + "Ambiguous native keyword call expansion" + ) + finish = index._span(node)[1] + begin = words._find_parens_start(finish - 1) + insertion = finish - 1 + existing = {keyword.arg: keyword.value for keyword in node.keywords} + added = [] + if fstrings is None: + fstrings = _fstring_quotes(pymodule) + quote = _enclosing_fstring_quote(fstrings, *index._span(node)) + for addition, requested in selected: + expression = _keyword_expression(requested) + value = existing.get(addition.name) + if value is not None: + if ast.dump(value) != ast.dump(expression): + raise rope.base.exceptions.RefactoringError( + "Conflicting existing keyword argument: " + + addition.name + ) + continue + added.append( + addition.name + "=" + _replacement_field_text(expression, quote) + ) + if not added: + continue + generator = ( + len(node.args) == 1 + and isinstance(node.args[0], ast.GeneratorExp) + and index._span(node.args[0]) == (begin, finish) + ) + if generator: + yield begin + 1, begin + 1, "(" + else: + if not self.required: + continue + raise rope.base.exceptions.RefactoringError( + "Signature consumer escapes as a non-call reference" + ) + prefix = real[begin + 1 : insertion].rstrip() + line_start = source.rfind("\n", begin + 1, insertion) + 1 + if line_start > begin and not source[line_start:insertion].strip(): + parameters = ( + tuple(existing.values()) + if occurrence.is_defined() + else (*node.args, *node.keywords) + ) + indentation = source[ + line_start:insertion + ] + " " * sourceutils.get_indent(pymodule.pycore.project) + for parameter in sorted( + parameters, key=lambda item: index._span(item)[0], reverse=True + ): + parameter_start = index._span(parameter)[0] + parameter_line = source.rfind("\n", begin + 1, parameter_start) + 1 + candidate = source[parameter_line:parameter_start].rstrip("*") + if parameter_line > begin and not candidate.strip(): + indentation = candidate + break + if prefix and not prefix.endswith(","): + last = begin + 1 + len(prefix) + yield last, last, (")" if generator else "") + "," + yield line_start, line_start, "".join( + indentation + item + ",\n" for item in added + ) + continue + separator = ( + ", " if prefix and not prefix.endswith(",") else " " if prefix else "" + ) + suffix = ( + ", " if occurrence.is_defined() and arguments.kwarg is not None else "" + ) + yield ( + insertion, + insertion, + (")" if generator else "") + separator + ", ".join(added) + suffix, + ) + + @staticmethod + def _check_decorators(owner, index): + decorators = owner.get_ast().decorator_list + if not decorators: + return + if ( + isinstance(owner, pyobjects.PyClass) + or len(decorators) != 1 + or isinstance(decorators[0], ast.Call) + ): + raise rope.base.exceptions.RefactoringError( + "Signature decorator has no proven native behavior" + ) + binding = index.decorator_bindings(owner.get_scope())[0] + if binding is None or not ( + getattr(binding.get_object(), "builtin", None) + in (staticmethod, classmethod) + or builtins._is_runtime_form( + binding.get_object(), contextlib.contextmanager + ) + ): + raise rope.base.exceptions.RefactoringError( + "Signature decorator has no proven native behavior" + ) + + def _refuse_opaque_consumer(self, pymodule, node): + if not isinstance(node, ast.Call): + return + scope = pymodule.get_scope().get_inner_scope_for_line(node.lineno) + callee = evaluate.eval_node(scope, node.func) + target = callee.get_object() if callee is not None else None + if ( + target is None + or target == pyobjects.get_unknown() + or occurrences.unsure_pyname(callee) + ): + raise rope.base.exceptions.RefactoringError( + "Unresolved callable cannot prove absence of an opaque signature consumer" + ) + if getattr(target, "builtin", None) in (eval, exec): + try: + if ( + not node.args + or not isinstance(node.args[0], ast.Constant) + or not isinstance(node.args[0].value, str) + ): + raise ValueError("Dynamic code has no literal source") + if target.builtin is eval: + ast.literal_eval(node.args[0].value) + else: + for statement in ast.parse(node.args[0].value).body: + if isinstance(statement, ast.Pass): + continue + if not isinstance(statement, ast.Expr): + raise ValueError("Dynamic statement can change bindings") + ast.literal_eval(statement.value) + except (SyntaxError, ValueError, TypeError) as error: + raise rope.base.exceptions.RefactoringError( + "Dynamic code has no proven signature consumer behavior" + ) from error + return + module = target.get_module() if target is not None else None + internal = module is not None and module.get_resource() in self.resources + for argument in (*node.args, *(keyword.value for keyword in node.keywords)): + for binding in self._argument_bindings(pymodule, argument, set()): + for value in self._contained_objects(binding.get_object()): + self._check_opaque_value(pymodule, value, internal) + + def _check_opaque_value(self, pymodule, value, internal): + if not internal and (value is None or value == pyobjects.get_unknown()): + raise rope.base.exceptions.RefactoringError( + "Unresolved value cannot prove absence of an opaque signature consumer" + ) + function = classfunctions.canonical_function(value) + module = function.get_module() + if isinstance(function, pyobjects.AbstractFunction) and not ( + module is not None + and module.get_resource() in self.resources + or isinstance(function, builtins.BuiltinFunction) + and function.function is not None + or builtins.has_runtime_call_model(function) + ): + raise rope.base.exceptions.RefactoringError( + "Unproved callable escapes through an opaque signature consumer" + ) + if internal: + return + if not isinstance(value, (pyobjects.AbstractClass, pyobjects.AbstractModule)): + value = value.get_type() + members = value.get_attributes() + if any( + occurrences.same_pyname(declared_binding, members.get(name)) + for name, declared_binding in self.reflective_targets + ): + raise rope.base.exceptions.RefactoringError( + "Target namespace escapes through an opaque signature consumer" + ) + + def _argument_bindings(self, pymodule, expression, seen): + identity = (id(pymodule), id(expression)) + if identity in seen: + return + seen.add(identity) + scope = pymodule.get_scope().get_inner_scope_for_line(expression.lineno) + binding = evaluate.eval_node(scope, expression) + if binding is not None: + yield binding + while isinstance(binding, pynames.ImportedName): + underlying = binding._get_imported_pyname() + if id(underlying) in seen: + break + seen.add(id(underlying)) + binding = underlying + if isinstance(binding, pynames.AssignedName): + for assignment in binding.assignments: + value = assignment.ast_node + for level in assignment.levels: + if not isinstance(value, (ast.List, ast.Tuple)) or level >= len( + value.elts + ): + value = None + break + value = value.elts[level] + if value is not None: + yield from self._argument_bindings(binding.module, value, seen) + if isinstance(expression, (ast.List, ast.Tuple, ast.Set)): + children = expression.elts + elif isinstance(expression, ast.Dict): + children = [ + item + for item in (*expression.keys, *expression.values) + if item is not None + ] + elif isinstance(expression, ast.Starred): + children = [expression.value] + elif isinstance(expression, ast.Subscript) and isinstance( + expression.slice, ast.Slice + ): + children = [expression.value] + else: + children = [] + for child in children: + yield from self._argument_bindings(pymodule, child, seen) + + def validate_effect(self, pymodule, function, args): + function = classfunctions.canonical_function(function) + module = function.get_module() + internal = module is not None and module.get_resource() in self.resources + names = ( + function.get_param_names() + if isinstance(function, pyobjects.AbstractFunction) + else [] + ) + for binding in args.get_pynames(names): + if binding is None: + continue + for value in self._contained_objects(binding.get_object()): + self._check_opaque_value(pymodule, value, internal) + + @staticmethod + def _contained_objects(value): + pending = [value] + seen = set() + while pending: + value = pending.pop() + if value is None or id(value) in seen: + continue + seen.add(id(value)) + if isinstance(value, pyobjects.SampledValue): + # A sampled read leaves its container position unproved; the + # element itself is the value an opaque consumer receives. + pending.append(value.representative) + continue + yield value + if isinstance(value, builtins.NamespaceItems): + pending.append(value.receiver.get_object()) + continue + kind = value.get_type() + if isinstance( + kind, + ( + builtins.Sequence, + builtins.List, + builtins.Set, + builtins.Iterator, + builtins.Generator, + ), + ): + pending.append(kind.holding) + elif isinstance(kind, builtins.Tuple): + pending.extend(kind.get_holding_objects()) + elif isinstance(kind, builtins.Dict): + pending.extend((kind.keys, kind.values)) + if isinstance(value, (pyobjects.PyClass, pyobjects.PyModule)): + owner = value + elif isinstance(kind, pyobjects.PyClass): + owner = kind + else: + continue + pending.extend( + member.get_object() + for member in owner.get_attributes().values() + if not isinstance(member.get_object(), pyobjects.AbstractFunction) + ) + + +class _SignatureEffects(soa.SOAVisitor): + def __init__(self, pycore, pydefined, follow_callback=None, *, proof): + super().__init__(pycore, pydefined, follow_callback) + self.proof = proof + self.node = None + + def visit(self, node): + previous = self.node + self.node = node + try: + return super().visit(node) + finally: + self.node = previous + + def _failure(self, operation, error): + resource = self.pymodule.get_resource() + raise rope.base.exceptions.RefactoringError( + f"{resource.path}:{self.node.lineno}: {operation}: {error}" + ) from error + + def _Call(self, node): + if self.proof.required: + try: + self.proof._refuse_opaque_consumer(self.pymodule, node) + except rope.base.exceptions.RefactoringError as error: + self._failure("call " + type(node.func).__name__, error) + super()._Call(node) + + def _call(self, function, args): + super()._call(function, args) + if self.proof.required and not ( + isinstance(self.node, (ast.BinOp, ast.AugAssign, ast.Compare)) + and isinstance(function, builtins.BuiltinFunction) + and function.parent is builtins.builtins["int"].get_object() + ): + try: + self.proof.validate_effect(self.pymodule, function, args) + except rope.base.exceptions.RefactoringError as error: + self._failure("effect " + str(function.get_name()), error) + + class _ChangeCallsInModule: def __init__(self, project, occurrence_finder, resource, call_changer): self.project = project @@ -323,13 +1106,36 @@ def __init__(self, project, occurrence_finder, resource, call_changer): def get_changed_module(self): word_finder = worder.Worder(self.source) change_collector = codeanalyze.ChangeCollector(self.source) + if isinstance(self.call_changer, _KeywordAdditions): + for start, end, replacement in self.call_changer.changes( + self.pymodule, self.occurrence_finder.find_occurrences(self.resource) + ): + change_collector.add_change(start, end, replacement) + changed = change_collector.get_changed() + if changed is not None: + try: + compile(changed, self.resource.path, "exec", dont_inherit=True) + except SyntaxError as error: + raise rope.base.exceptions.RefactoringError( + "Keyword addition is invalid in the native source context" + ) from error + return changed for occurrence in self.occurrence_finder.find_occurrences(self.resource): if not occurrence.is_called() and not occurrence.is_defined(): continue + primary, pyname = occurrence.get_primary_and_pyname() + if pyname is not None: + pyfunction = pyname.get_object() + if ( + isinstance(pyfunction, pyobjects.PyClass) + and "__init__" in pyfunction + ): + pyfunction = pyfunction["__init__"].get_object() + if isinstance(pyfunction, pyobjects.PyFunction): + functionutils._check_signature_parameters(pyfunction) start, end = occurrence.get_primary_range() begin_parens, end_parens = word_finder.get_word_parens_range(end - 1) if occurrence.is_called(): - primary, pyname = occurrence.get_primary_and_pyname() changed_call = self.call_changer.change_call( primary, pyname, self.source[start:end_parens] ) diff --git a/rope/refactor/extract.py b/rope/refactor/extract.py index ea373613d..5d97355cc 100644 --- a/rope/refactor/extract.py +++ b/rope/refactor/extract.py @@ -206,7 +206,27 @@ def global_(self): @property def method(self): - return self.scope.parent is not None and self.scope.parent.get_kind() == "Class" + return ( + self.scope.get_kind() == "Function" + and self.scope.parent is not None + and self.scope.parent.get_kind() == "Class" + ) + + @property + def class_to_define_before(self): + """The class the extracted function must be defined before, if any + + Class bodies run when the class is defined, so a function extracted + from a class body has to be defined before the (outermost) class. + """ + if self.scope.get_kind() != "Class": + return None + scope = self.scope + while scope.parent.get_kind() == "Class": + scope = scope.parent + if self.make_global and scope.parent.get_kind() != "Module": + return None + return scope @property def indents(self): @@ -400,6 +420,8 @@ def __init__(self, info, matched_lines): def find_lineno(self): if self.info.variable and not self.info.make_global: return self._get_before_line() + if self.info.class_to_define_before is not None: + return self._get_before_class() if self.info.global_: toplevel = self._find_toplevel(self.info.scope) ast = self.info.pymodule.get_ast() @@ -420,6 +442,10 @@ def _find_toplevel(self, scope): def find_indents(self): if self.info.variable and not self.info.make_global: return sourceutils.get_indents(self.info.lines, self._get_before_line()) + if self.info.class_to_define_before is not None: + return sourceutils.get_indents( + self.info.lines, self.info.class_to_define_before.get_start() + ) else: if self.info.global_ or self.info.make_global: return 0 @@ -432,6 +458,10 @@ def _get_before_line(self): def _get_after_scope(self): return self.info.scope.get_end() + 1 + def _get_before_class(self): + node = self.info.class_to_define_before.pyobject.get_ast() + return min([node.lineno] + [d.lineno for d in node.decorator_list]) + class _ExceptionalConditionChecker: def __call__(self, info): @@ -489,14 +519,6 @@ def multi_line_conditions(self, info): ) if usefunction._yield_count(node): raise RefactoringError("Extracted piece cannot have yield statements.") - if not hasattr( - ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT" - ) and _AsyncStatementFinder.has_errors(extracted): - raise RefactoringError( - "Extracted piece can only have async/await " - "statements if Rope is running on Python " - "3.8 or higher" - ) if count == 1 and not usefunction._returns_last(node): raise RefactoringError("Return should be the last statement.") if info.region != info.lines_region: @@ -558,7 +580,7 @@ def _extracting_from_classmethod(self): return self.info.method and _get_function_kind(self.info.scope) == "classmethod" def get_definition(self): - if self.info.global_: + if self.info.global_ or self.info.class_to_define_before is not None: return "\n%s\n" % self._get_function_definition() else: return "\n%s" % self._get_function_definition() @@ -904,7 +926,12 @@ def _AugAssign(self, node): self.visit(node.target) def _ClassDef(self, node): - self._written_variable(node.name, node.lineno) + if not self.is_global and self.host_function: + self.host_function = False + for child in node.body: + self.visit(child) + else: + self._written_variable(node.name, node.lineno) def _ListComp(self, node): self._comp_exp(node) @@ -1090,23 +1117,6 @@ def _ClassDef(self, node): pass -class _AsyncStatementFinder(_BaseErrorFinder): - def __init__(self): - self.error = False - - def _AsyncFor(self, node): - self.error = True - - def _AsyncWith(self, node): - self.error = True - - def _FunctionDef(self, node): - pass - - def _ClassDef(self, node): - pass - - class _UnbalancedRegionFinder(_BaseErrorFinder): """ Flag an error if we are including the start of a block without the end. diff --git a/rope/refactor/functionutils.py b/rope/refactor/functionutils.py index 564e77beb..0a2d42441 100644 --- a/rope/refactor/functionutils.py +++ b/rope/refactor/functionutils.py @@ -1,34 +1,64 @@ from typing import List, Tuple -from rope.base import ast, pyobjects, worder -from rope.base.builtins import Lambda +from rope.base import ast, exceptions, pyobjects, worder +from rope.base.builtins import Lambda, bound_receiver from rope.base.codeanalyze import ASTLinesAdapter +def _get_function_signature(pyfunction): + """Read the original signature without using the legacy editable model.""" + pymodule = pyfunction.get_module() + word_finder = worder.Worder(pymodule.source_code) + start = pymodule.lines.get_line_start(pyfunction.get_ast().lineno) + if isinstance(pyfunction, Lambda): + return word_finder.get_lambda_and_args(start) + return word_finder.get_function_and_args_in_header(start) + + +def _check_signature_parameters(pyfunction): + """Refuse parameter kinds the legacy signature writer cannot preserve.""" + arguments = pyfunction.get_ast().args + if arguments.posonlyargs or arguments.kwonlyargs: + raise exceptions.RefactoringError( + "This refactoring does not support positional-only or keyword-only parameters." + ) + + class DefinitionInfo: def __init__( - self, function_name, is_method, args_with_defaults, args_arg, keywords_arg + self, + function_name, + is_method, + args_with_defaults, + args_arg, + keywords_arg, + annotations=None, ): self.function_name = function_name self.is_method = is_method self.args_with_defaults = args_with_defaults self.args_arg = args_arg self.keywords_arg = keywords_arg + self.annotations = annotations or {} - def to_string(self): - return f"{self.function_name}({self.arguments_to_string()})" + def to_string(self, *, annotated=False): + return f"{self.function_name}({self.arguments_to_string(annotated=annotated)})" + + def arguments_to_string(self, from_index=0, *, annotated=False): + def parameter(name): + annotation = self.annotations.get(name) if annotated else None + return f"{name}: {annotation}" if annotation is not None else name - def arguments_to_string(self, from_index=0): params = [] for arg, default in self.args_with_defaults: if default is not None: - params.append(f"{arg}={default}") + params.append(f"{parameter(arg)}={default}") else: - params.append(arg) + params.append(parameter(arg)) if self.args_arg is not None: - params.append("*" + self.args_arg) + params.append("*" + parameter(self.args_arg)) if self.keywords_arg: - params.append("**" + self.keywords_arg) + params.append("**" + parameter(self.keywords_arg)) return ", ".join(params[from_index:]) @staticmethod @@ -54,19 +84,16 @@ def _read(pyfunction, code): args_with_defaults, args_arg, keywords_arg, + { + argument.arg: info._get_source_range(argument.annotation) + for argument in ast.walk(info.ast.args) + if isinstance(argument, ast.arg) and argument.annotation is not None + }, ) @staticmethod def read(pyfunction): - pymodule = pyfunction.get_module() - word_finder = worder.Worder(pymodule.source_code) - lineno = pyfunction.get_ast().lineno - start = pymodule.lines.get_line_start(lineno) - if isinstance(pyfunction, Lambda): - call = word_finder.get_lambda_and_args(start) - else: - call = word_finder.get_function_and_args_in_header(start) - return DefinitionInfo._read(pyfunction, call) + return DefinitionInfo._read(pyfunction, _get_function_signature(pyfunction)) class CallInfo: @@ -137,7 +164,10 @@ def read(primary, pyname, definition_info, code): def _is_method_call(primary, pyname): return ( primary is not None - and isinstance(primary.get_object().get_type(), pyobjects.PyClass) + and isinstance( + bound_receiver(primary.get_object()).get_type(), + pyobjects.PyClass, + ) and CallInfo._is_method(pyname) ) @@ -231,7 +261,8 @@ def is_called_as_a_method(self): def _get_source_range(self, tree: ast.AST) -> str: source = self._lines.get_source_segment(tree) - assert source is not None + if source is None: + raise exceptions.RefactoringError("Missing native function source range") return source @@ -250,17 +281,16 @@ def get_parameters(self) -> Tuple[List[str], List[Tuple[str, str]]]: # properly to handle them properly at some point args = [] kwargs = [] - args += [arg.arg for arg in self.ast.args.posonlyargs] - args += [arg.arg for arg in self.ast.args.args] + positional = self.ast.args.posonlyargs + self.ast.args.args + defaults = self.ast.args.defaults + required = len(positional) - len(defaults) + args += [arg.arg for arg in positional[:required]] + kwargs += [ + (argument.arg, self._get_source_range(value)) + for argument, value in zip(positional[required:], defaults) + ] if self.ast.args.vararg is not None: args += ["*" + self.ast.args.vararg.arg] - if len(self.ast.args.defaults) > 0: - defaults = self.ast.args.defaults - kwargs += [ - (name, self._get_source_range(value)) - for name, value in zip(args[-len(defaults) :], defaults) - ] - del args[-len(self.ast.args.defaults) :] if self.ast.args.kwarg is not None: args += ["**" + self.ast.args.kwarg.arg] if self.ast.args.kwonlyargs: diff --git a/rope/refactor/importutils/actions.py b/rope/refactor/importutils/actions.py index 4d748ba52..a4a5fdde2 100644 --- a/rope/refactor/importutils/actions.py +++ b/rope/refactor/importutils/actions.py @@ -175,6 +175,11 @@ def visitFromImport(self, import_stmt, import_info): ): if import_info.is_star_import(): return True + if import_stmt.readonly: + return not self.import_info.is_star_import() and all( + pair in import_info.names_and_aliases + for pair in self.import_info.names_and_aliases + ) if self.import_info.is_star_import(): import_stmt.import_info = self.import_info return True @@ -341,6 +346,7 @@ def __init__(self, project, pymodule, pyname, folder): self.pymodule = pymodule self.pyname = pyname self.context = importinfo.ImportContext(project, folder) + self.removed_names = [] def visitFromImport(self, import_stmt, import_info): if import_info.is_star_import(): @@ -351,6 +357,7 @@ def visitFromImport(self, import_stmt, import_info): try: pyname = self.pymodule[alias or name] if occurrences.same_pyname(self.pyname, pyname): + self.removed_names.append(alias or name) continue except exceptions.AttributeNotFoundError: pass diff --git a/rope/refactor/importutils/module_imports.py b/rope/refactor/importutils/module_imports.py index f35b22521..206d958ee 100644 --- a/rope/refactor/importutils/module_imports.py +++ b/rope/refactor/importutils/module_imports.py @@ -1,6 +1,6 @@ from typing import List, Union -from rope.base import ast, exceptions, pynames, pynamesdef, utils +from rope.base import ast, exceptions, pynames, pynamesdef, pyobjects, utils from rope.refactor.importutils import actions, importinfo @@ -25,8 +25,9 @@ def imports(self): return result def _get_unbound_names(self, defined_pyobject): - visitor = _GlobalUnboundNameFinder(self.pymodule, defined_pyobject) - visitor.visit(self.pymodule.get_ast()) + source_module = defined_pyobject.get_module() + visitor = _GlobalUnboundNameFinder(source_module, defined_pyobject) + visitor.visit(source_module.get_ast()) return visitor.unbound def _get_all_star_list(self, pymodule): @@ -114,7 +115,7 @@ def remove_unused_imports(self): def get_used_imports(self, defined_pyobject): result = [] - can_select = _OneTimeSelector(self._get_unbound_names(defined_pyobject)) + can_select = _OneTimeSelector(self.get_used_names(defined_pyobject)) visitor = actions.FilteringVisitor( self.project, self._current_folder(), can_select ) @@ -124,6 +125,30 @@ def get_used_imports(self, defined_pyobject): result.append(new_import) return result + def get_used_names(self, defined_pyobject): + """Return lexical references and a module's declared export names.""" + names = self._get_unbound_names(defined_pyobject) + from rope.base import evaluate + from rope.refactor import occurrences + + module = defined_pyobject.get_module() + index = module.get_source_binding_index() + scope = defined_pyobject.get_scope() + for name, binding in module._get_structural_attributes().items(): + if not isinstance(binding, (pynames.ImportedName, pynames.ImportedModule)): + continue + for reference in index.quoted_annotation_references(name): + line = module.lines.get_line_number(reference[0]) + if scope.get_start() <= line <= scope.get_end(): + actual = evaluate.eval_str(reference[4].environment, name) + if occurrences.same_pyname(binding, actual): + names.add(name) + break + if isinstance(defined_pyobject, pyobjects.AbstractModule): + # A package exports through its ``__init__`` module. + names |= self._get_all_star_list(defined_pyobject.get_module()) + return names + def get_changed_source(self): if not self.project.prefs.get("pull_imports_to_top") and not self.sorted: return "".join(self._rewrite_imports(self.imports)) @@ -223,6 +248,9 @@ def _get_new_import_blanks(self): def _get_new_import_lineno(self): if self.imports: return self.imports[-1].end_line + for node in self.pymodule.get_ast().body: + if isinstance(node, ast.ImportFrom) and node.module == "__future__": + return node.lineno + 1 return 1 def filter_names(self, can_select): @@ -247,8 +275,7 @@ def remove_duplicates(self): for added_import in added_imports: if added_import.accept(visitor): import_stmt.empty_import() - else: - added_imports.append(import_stmt) + added_imports.append(import_stmt) def force_single_imports(self): """force a single import per statement""" @@ -373,12 +400,16 @@ def handle_long_imports(self, maxdots, maxlength): return visitor.to_be_renamed def remove_pyname(self, pyname): - """Removes pyname when imported in ``from mod import x``""" + """Remove pyname when imported in ``from mod import x``. + + Returns the names that were actually removed. + """ visitor = actions.RemovePyNameVisitor( self.project, self.pymodule, pyname, self._current_folder() ) for import_stmt in self.imports: import_stmt.accept(visitor) + return visitor.removed_names def get_first_decorator_or_function_start_line(node): diff --git a/rope/refactor/inline.py b/rope/refactor/inline.py index 768d30b7d..22c102c68 100644 --- a/rope/refactor/inline.py +++ b/rope/refactor/inline.py @@ -17,10 +17,12 @@ # but it should be 200. import re +import sys from typing import List import rope.base.builtins # Use fully qualified names for clarity. from rope.base import ( + ast, codeanalyze, evaluate, exceptions, @@ -38,6 +40,7 @@ importutils, move, occurrences, + patchedast, rename, sourceutils, ) @@ -239,12 +242,23 @@ def get_kind(self): class InlineVariable(_Inliner): def __init__(self, *args, **kwds): super().__init__(*args, **kwds) + if not isinstance(self.pyname, pynames.AssignedName): + raise exceptions.RefactoringError( + "Inline variable should be performed on an assigned variable." + ) self.pymodule = self.pyname.get_definition_location()[0] self.resource = self.pymodule.get_resource() self._check_exceptional_conditions() self._init_imports() def _check_exceptional_conditions(self): + if any( + isinstance(assignment.ast_node, ast.pattern) + for assignment in self.pyname.assignments + ): + raise exceptions.RefactoringError( + "Pattern matching captures cannot be inlined." + ) if len(self.pyname.assignments) != 1: raise exceptions.RefactoringError( "Local variable should be assigned once for inlining." @@ -267,6 +281,16 @@ def get_changes( resources = [self.original] if remove and self.original != self.resource: resources.append(self.resource) + resources = list(resources) + check_resources = resources + if remove and not rename._is_local(self.pyname): + check_resources = list( + dict.fromkeys(resources + list(self.project.get_python_files())) + ) + for resource in check_resources: + self._check_deferred_annotation_references(resource, remove, only_current) + if remove and self.resource in resources: + self._check_initializer_removal(resources, only_current) changes = ChangeSet("Inline variable <%s>" % self.name) jobset = task_handle.create_jobset("Calculating changes", len(resources)) @@ -283,6 +307,303 @@ def get_changes( jobset.finished_job() return changes + def _check_deferred_annotation_references(self, resource, remove, only_current): + pymodule = self.project.get_pymodule(resource) + tree = pymodule.get_ast() + future_annotations = any( + isinstance(node, ast.ImportFrom) + and node.module == "__future__" + and any(alias.name == "annotations" for alias in node.names) + for node in tree.body + ) + if not future_annotations and sys.version_info < (3, 12): + return + annotations = [] + type_parameters = [] + + def collect(node, scope=None): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + parameters = getattr(node, "type_params", []) + if parameters: + namespace = ( + pymodule.get_scope() + .get_inner_scope_for_line(node.lineno) + .parent + ) + names = {parameter.name for parameter in parameters} + for parameter in parameters: + for attr in ("bound", "default_value"): + expression = getattr(parameter, attr, None) + if expression is not None: + type_parameters.append((expression, namespace, names)) + if isinstance(node, ast.arg): + annotations.append(node.annotation) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + annotations.append(node.returns) + scope = "function" + elif isinstance(node, ast.Lambda): + scope = "function" + elif isinstance(node, ast.ClassDef): + scope = "class" + elif ( + isinstance(node, ast.AnnAssign) and node.simple and scope != "function" + ): + annotations.append(node.annotation) + for child in ast.iter_child_nodes(node): + collect(child, scope) + + collect(tree) + if not future_annotations and sys.version_info < (3, 14): + annotations = [] + lines = codeanalyze.ASTLinesAdapter(pymodule.source_code) + regions = [lines[node] for node in annotations if node is not None] + parameter_regions = [lines[item[0]] for item in type_parameters] + if not regions and not parameter_regions: + return + + def parameter_reference(node, namespace, names): + inner_namespace = namespace + inner_names = set(names) + while inner_namespace.get_kind() == "Class": + inner_names.update( + parameter.name + for parameter in getattr( + inner_namespace.pyobject.get_ast(), "type_params", [] + ) + ) + inner_namespace = inner_namespace.parent + if isinstance(node, ast.Lambda): + arguments = node.args + defaults = arguments.defaults + [ + default for default in arguments.kw_defaults if default is not None + ] + bound_names = inner_names | { + argument.arg + for argument in ( + arguments.posonlyargs + + arguments.args + + arguments.kwonlyargs + + [arguments.vararg, arguments.kwarg] + ) + if argument is not None + } + return any( + parameter_reference(default, namespace, names) + for default in defaults + ) or parameter_reference(node.body, inner_namespace, bound_names) + if isinstance( + node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp) + ): + bound_names = set(names) + for index, generator in enumerate(node.generators): + iterable_namespace = namespace if index == 0 else inner_namespace + if parameter_reference( + generator.iter, iterable_namespace, bound_names + ): + return True + bound_names.update(inner_names) + bound_names.update( + name.id + for name in ast.walk(generator.target) + if isinstance(name, ast.Name) + and isinstance(name.ctx, ast.Store) + ) + if parameter_reference( + generator.target, inner_namespace, bound_names + ): + return True + if any( + parameter_reference(condition, inner_namespace, bound_names) + for condition in generator.ifs + ): + return True + values = ( + [node.key, node.value] + if isinstance(node, ast.DictComp) + else [node.elt] + ) + return any( + parameter_reference(value, inner_namespace, bound_names) + for value in values + ) + if isinstance(node, (ast.Name, ast.Attribute)): + primary = node + while isinstance(primary, ast.Attribute): + primary = primary.value + if not (isinstance(primary, ast.Name) and primary.id in names): + if isinstance(primary, ast.Name): + outer_scope = namespace + while outer_scope is not None: + if ( + outer_scope.get_kind() != "Class" + or outer_scope is namespace + ) and primary.id in outer_scope.get_names(): + break + if any( + parameter.name == primary.id + for parameter in getattr( + outer_scope.pyobject.get_ast(), "type_params", [] + ) + ): + return False + outer_scope = outer_scope.parent + if occurrences.same_pyname( + self.pyname, evaluate.eval_node(namespace, node) + ): + return True + return any( + parameter_reference(child, namespace, names) + for child in ast.iter_child_nodes(node) + ) + + if remove and any( + parameter_reference(expression, namespace, names) + for expression, namespace, names in type_parameters + ): + # Type parameter expressions use the enclosing annotation scope, + # not the function's parameters/locals or the new class's body. + raise exceptions.RefactoringError( + "Cannot inline a variable referenced in a deferred annotation." + ) + module_names = pymodule.get_scope().get_names() + + def runtime_annotation_eval(node): + """Resolve a deferred annotation expression the way Python does. + + ``typing.get_type_hints`` evaluates class annotations with the + module globals ahead of class and function locals (it swaps the + two namespaces for classes), while the source binding index + resolves the annotation in its own annotation environment. A + module-level binding therefore wins over shadowing names in the + class or function header, and only names without a module-level + binding fall back to the annotation environment. + """ + if isinstance(node, ast.Attribute): + base = runtime_annotation_eval(node.value) + if base is None: + return None + pyobject = base.get_object() + if pyobject == pyobjects.get_unknown(): + return None + try: + return pyobject[node.attr] + except exceptions.AttributeNotFoundError: + return None + if isinstance(node, ast.Name): + if node.id in module_names: + return module_names[node.id] + return evaluate.eval_node(pymodule.get_scope(), node) + return None + + if ( + future_annotations + and remove + and any( + isinstance(node, (ast.Name, ast.Attribute)) + and occurrences.same_pyname(self.pyname, runtime_annotation_eval(node)) + for annotation in annotations + if annotation is not None + for node in ast.walk(annotation) + ) + ): + # Stringified annotations may resolve module globals even when + # static name lookup finds a class member or function parameter. + raise exceptions.RefactoringError( + "Cannot inline a variable referenced in a deferred annotation." + ) + finder = occurrences.create_finder( + self.project, self.name, self.pyname, imports=False + ) + for occurrence in finder.find_occurrences(pymodule=pymodule): + current = True + if only_current: + start, end = occurrence.get_primary_range() + current = resource == self.original and start <= self.offset <= end + if not remove and not current: + continue + start, _ = occurrence.get_word_range() + if any(begin <= start < end for begin, end in regions) or ( + current + and any(begin <= start < end for begin, end in parameter_regions) + ): + # Substituting the initializer can change when an annotation + # evaluates it and which bindings it observes. + raise exceptions.RefactoringError( + "Cannot inline a variable referenced in a deferred annotation." + ) + + def _check_initializer_removal(self, resources, only_current): + initializer = self.pyname.assignments[0].ast_node + statement = next( + ( + node + for node in ast.walk(self.pymodule.get_ast()) + if isinstance(node, (ast.Assign, ast.AnnAssign)) + and node.value is initializer + ), + None, + ) + statement_region = ( + codeanalyze.ASTLinesAdapter(self.pymodule.source_code)[statement] + if statement is not None + else None + ) + local_annotation = isinstance(statement, ast.AnnAssign) and ( + isinstance(statement.target, ast.Name) + and self.pymodule.get_scope() + .get_inner_scope_for_line(statement.lineno) + .get_kind() + == "Function" + ) + if isinstance(statement, ast.AnnAssign) and not local_annotation: + raise exceptions.RefactoringError( + "Cannot discard a variable annotation while inlining." + ) + bound_name = isinstance(initializer, ast.Name) and _is_bound_initializer_name( + self.pymodule, statement, initializer + ) + if isinstance(initializer, ast.Name) and not bound_name: + raise exceptions.RefactoringError( + "Cannot discard an initializer name that may be unbound." + ) + finder = occurrences.create_finder( + self.project, self.name, self.pyname, imports=False + ) + has_reads = False + for resource in self.project.get_python_files(): + for occurrence in finder.find_occurrences(resource=resource): + start, _ = occurrence.get_primary_range() + if ( + resource == self.resource + and statement_region is not None + and (statement_region[0] <= start < statement_region[1]) + ): + continue + if occurrence.is_written() or occurrence.is_a_fixed_primary(): + continue + has_reads = True + if resource not in resources or ( + only_current + and resource == self.original + and not self.region[0] <= start < self.region[1] + ): + raise exceptions.RefactoringError( + "Cannot remove a variable referenced outside the selected occurrences." + ) + if has_reads: + return + plain_assignment = ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Name) + ) + if not (plain_assignment or local_annotation) or not ( + bound_name or _is_pure_unused_initializer(initializer) + ): + raise exceptions.RefactoringError( + "Cannot discard an unused variable initializer while inlining." + ) + def _change_main_module(self, remove, only_current, docs): region = None if only_current and self.original == self.resource: @@ -295,6 +616,7 @@ def _change_main_module(self, remove, only_current, docs): remove=remove, region=region, docs=docs, + check_lambda=True, ) def _init_imports(self): @@ -314,10 +636,13 @@ def check_aim(occurrence): filters.insert(0, check_aim) finder = occurrences.Finder(self.project, self.name, filters=filters) + _check_lambda_occurrences(finder, self.project.get_pymodule(resource)) changed = rename.rename_in_module( finder, self.imported, resource=resource, replace_primary=True ) - if changed and remove: + if remove: + # An importing module without value reads still needs its import + # cleaned up once the definition is discarded. changed = _remove_from(self.project, self.pyname, changed, resource) return changed @@ -328,8 +653,28 @@ def get_kind(self): class InlineParameter(_Inliner): def __init__(self, *args, **kwds): super().__init__(*args, **kwds) + if not isinstance(self.pyname, pynames.ParameterName): + raise exceptions.RefactoringError( + "Inline parameter should be performed on a parameter." + ) + pyfunction = self.pyname.pyfunction + functionutils._check_signature_parameters(pyfunction) + if self.name not in [ + argument.arg for argument in pyfunction.get_ast().args.args + ]: + raise exceptions.RefactoringError( + "Cannot inline the default of a list or keyword argument." + ) + definition_info = functionutils.DefinitionInfo.read(pyfunction) + indices = [ + index + for index, (name, default) in enumerate(definition_info.args_with_defaults) + if name == self.name + ] + if len(indices) != 1: + raise exceptions.RefactoringError("Cannot resolve the parameter default.") resource, offset = self._function_location() - index = self.pyname.index + index = indices[0] self.changers = [change_signature.ArgumentDefaultInliner(index)] self.signature = change_signature.ChangeSignature( self.project, resource, offset @@ -393,6 +738,11 @@ def __init__(self, project, pyfunction, body=None): self.body = sourceutils.get_body(self.pyfunction) def _get_definition_info(self): + arguments = self.pyfunction.get_ast().args + if arguments.kwonlyargs or arguments.vararg or arguments.kwarg: + raise exceptions.RefactoringError( + "Cannot inline functions with list and keyword arguments." + ) return functionutils.DefinitionInfo.read(self.pyfunction) def _get_definition_params(self): @@ -426,6 +776,13 @@ def _calculate_header(self, primary, pyname, call): call_info = functionutils.CallInfo.read( primary, pyname, self.definition_info, call ) + positional_only = { + argument.arg for argument in self.pyfunction.get_ast().args.posonlyargs + } + if any(name in positional_only for name, value in call_info.keywords): + raise exceptions.RefactoringError( + "Cannot inline a positional-only parameter passed as a keyword." + ) paramdict = self.definition_params mapping = functionutils.ArgumentMapping(self.definition_info, call_info) for param_name, value in mapping.param_dict.items(): @@ -619,12 +976,38 @@ def lines(self): def _inline_variable( - project, pymodule, pyname, name, remove=True, region=None, docs=False + project, + pymodule, + pyname, + name, + remove=True, + region=None, + docs=False, + check_lambda=False, ): definition = _getvardef(pymodule, pyname) start, end = _assigned_lineno(pymodule, pyname) + assignment_node = _find_assignment_node(pymodule, pyname.assignments[0].ast_node) occurrence_finder = occurrences.create_finder(project, name, pyname, docs=docs) + if check_lambda: + _check_lambda_occurrences( + occurrence_finder, pymodule, region=region, writes=False + ) + if remove: + removal = _get_assignment_removal( + pymodule.source_code, + assignment_node, + name, + start, + end, + occurrence_finder, + pymodule, + ) + return _rename_and_remove_assignment( + occurrence_finder, definition, pymodule, removal, region + ) + changed_source = rename.rename_in_module( occurrence_finder, definition, @@ -633,34 +1016,276 @@ def _inline_variable( writes=False, region=region, ) - if changed_source is None: - changed_source = pymodule.source_code - if remove: - lines = codeanalyze.SourceLinesAdapter(changed_source) - source = ( - changed_source[: lines.get_line_start(start)] - + changed_source[lines.get_line_end(end) + 1 :] + return changed_source if changed_source is not None else pymodule.source_code + + +def _is_bound_initializer_name(pymodule, statement, initializer): + if statement is None: + return False + scope = pymodule.get_scope().get_inner_scope_for_line(statement.lineno) + if scope.get_kind() == "Class": + # Class-body aliasing stays conservative, but a builtin initializer + # name is always bound wherever it is substituted. + return isinstance( + evaluate.eval_node(scope, initializer), rope.base.builtins.BuiltinName ) - else: - source = changed_source - return source + pyname = evaluate.eval_node(scope, initializer) + if any( + isinstance(node, ast.Name) + and node.id == initializer.id + and isinstance(node.ctx, ast.Del) + and evaluate.eval_node( + pymodule.get_scope().get_inner_scope_for_line(node.lineno), node + ) + is pyname + for node in ast.walk(pymodule.get_ast()) + ): + return False + if isinstance(pyname, rope.base.builtins.BuiltinName): + return True + if isinstance(pyname, pynames.ParameterName): + return True + parents = { + child: node + for node in ast.walk(pymodule.get_ast()) + for child in ast.iter_child_nodes(node) + } + anchor = statement + while anchor in parents: + owner = parents[anchor] + for _, block in ast.iter_fields(owner): + if not isinstance(block, list) or anchor not in block: + continue + for previous in reversed(block[: block.index(anchor)]): + binding_scope = pymodule.get_scope().get_inner_scope_for_line( + previous.lineno + ) + if ( + isinstance(previous, ast.Assign) + and len(previous.targets) == 1 + and isinstance(previous.targets[0], ast.Name) + ): + if previous.targets[0].id == initializer.id: + if ( + evaluate.eval_node(binding_scope, previous.targets[0]) + is pyname + ): + return True + break + if _is_pure_unused_initializer(previous.value): + continue + if isinstance( + previous, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + if ( + previous.name == initializer.id + and pyname is not None + and pyname.get_definition_location() + == (pymodule, previous.lineno) + ): + return True + if isinstance(previous, (ast.FunctionDef, ast.AsyncFunctionDef)): + arguments = previous.args + parameters = ( + arguments.posonlyargs + + arguments.args + + arguments.kwonlyargs + + [arguments.vararg, arguments.kwarg] + ) + if not ( + previous.decorator_list + or arguments.defaults + or any(arguments.kw_defaults) + or previous.returns + or any( + parameter.annotation + for parameter in parameters + if parameter is not None + ) + or getattr(previous, "type_params", ()) + ): + continue + break + if isinstance(previous, (ast.Import, ast.ImportFrom)): + matching = any( + (alias.asname or alias.name.split(".")[0]) == initializer.id + for alias in previous.names + ) + if ( + matching + and evaluate.eval_node(binding_scope, initializer) is pyname + ): + return True + break + if isinstance(previous, ast.Pass): + continue + break + anchor = owner + return False + + +def _is_pure_unused_initializer(node): + # literal_eval accepts set(); even that name may be rebound to user code. + if any(isinstance(child, ast.Call) for child in ast.walk(node)): + return False + try: + ast.literal_eval(node) + except (TypeError, ValueError): + return False + return True def _getvardef(pymodule, pyname): assignment = pyname.assignments[0] + if assignment.levels: + raise exceptions.RefactoringError("Cannot inline tuple assignments.") + + assignment_node = _find_assignment_node(pymodule, assignment.ast_node) + if assignment_node is not None: + source = pymodule.source_code + node_start, node_end = patchedast.node_region(assignment_node) + value_start, _ = patchedast.node_region(assignment.ast_node) + separator = source.rfind("=", node_start, value_start) + if separator != -1: + return source[separator + 1 : node_end].strip() + lines = pymodule.lines start, end = _assigned_lineno(pymodule, pyname) definition_with_assignment = _join_lines( [lines.get_line(n) for n in range(start, end + 1)], ) - if assignment.levels: - raise exceptions.RefactoringError("Cannot inline tuple assignments.") definition = definition_with_assignment[ definition_with_assignment.index("=") + 1 : ].strip() return definition +def _find_assignment_node(pymodule, value_node): + if value_node is None: + return None + module_ast = pymodule.get_ast() + patchedast.patch_ast(module_ast, pymodule.source_code) + for node in ast.walk(module_ast): + if isinstance(node, (ast.Assign, ast.AnnAssign)) and node.value is value_node: + return node + + +def _rename_and_remove_assignment(finder, definition, pymodule, removal, region): + source = pymodule.source_code + remove_start, remove_end, replacement = removal + change_collector = codeanalyze.ChangeCollector(source) + for occurrence in finder.find_occurrences(pymodule=pymodule): + if occurrence.is_a_fixed_primary() or occurrence.is_written(): + continue + occurrence_start, occurrence_end = occurrence.get_primary_range() + if occurrence_start < remove_end and occurrence_end > remove_start: + continue + if region is None or region[0] <= occurrence_start < region[1]: + change_collector.add_change(occurrence_start, occurrence_end, definition) + change_collector.add_change(remove_start, remove_end, replacement) + changed_source = change_collector.get_changed() + return changed_source if changed_source is not None else source + + +def _get_assignment_removal( + source, assignment_node, name, start, end, finder, pymodule +): + if assignment_node is None: + return _get_logical_line_removal(source, start, end) + + targets = ( + assignment_node.targets + if isinstance(assignment_node, ast.Assign) + else [assignment_node.target] + ) + target_index = _find_assignment_target_index(targets, name, finder, pymodule) + if target_index is None: + return _get_logical_line_removal(source, start, end) + + if len(targets) > 1: + following_nodes = [*targets[1:], assignment_node.value] + segment_ends = [] + for target, following in zip(targets, following_nodes): + _, target_end = patchedast.node_region(target) + following_start, _ = patchedast.node_region(following) + separator = source.find("=", target_end, following_start) + if separator == -1: + return _get_logical_line_removal(source, start, end) + segment_ends.append( + _skip_assignment_gap(source, separator + 1, following_start) + ) + + node_start, _ = patchedast.node_region(assignment_node) + segment_starts = [node_start, *segment_ends[:-1]] + return segment_starts[target_index], segment_ends[target_index], "" + + node_start, node_end = patchedast.node_region(assignment_node) + lines = codeanalyze.SourceLinesAdapter(source) + line_start = lines.get_line_start(assignment_node.lineno) + line_end = lines.get_line_end(assignment_node.end_lineno) + + remove_end = node_end + while remove_end < line_end and source[remove_end] in " \t": + remove_end += 1 + if remove_end < line_end and source[remove_end] == ";": + remove_end += 1 + while remove_end < line_end and source[remove_end] in " \t": + remove_end += 1 + return node_start, remove_end, "" + + remove_start = node_start + while remove_start > line_start and source[remove_start - 1] in " \t": + remove_start -= 1 + if remove_start > line_start and source[remove_start - 1] == ";": + remove_start -= 1 + while remove_start > line_start and source[remove_start - 1] in " \t": + remove_start -= 1 + return remove_start, node_end, "" + + return _get_logical_line_removal(source, start, end) + + +def _target_matches_name(target, name): + return (isinstance(target, ast.Name) and target.id == name) or ( + isinstance(target, ast.Attribute) and target.attr == name + ) + + +def _find_assignment_target_index(targets, name, finder, pymodule): + for occurrence in finder.find_occurrences(pymodule=pymodule): + occurrence_start, occurrence_end = occurrence.get_word_range() + for index, target in enumerate(targets): + target_start, target_end = patchedast.node_region(target) + if target_start <= occurrence_start and occurrence_end <= target_end: + return index + + matching_indexes = [ + index + for index, target in enumerate(targets) + if _target_matches_name(target, name) + ] + if len(matching_indexes) == 1: + return matching_indexes[0] + + +def _skip_assignment_gap(source, offset, limit): + while offset < limit: + if source[offset] in " \t\r\n": + offset += 1 + elif source.startswith("\\\r\n", offset): + offset += 3 + elif source.startswith("\\\n", offset): + offset += 2 + else: + break + return offset + + +def _get_logical_line_removal(source, start, end): + lines = codeanalyze.SourceLinesAdapter(source) + return lines.get_line_start(start), lines.get_line_end(end) + 1, "" + + def _assigned_lineno(pymodule, pyname): definition_line = pyname.assignments[0].ast_node.lineno return pymodule.logical_lines.logical_line_in(definition_line) @@ -681,14 +1306,136 @@ def _add_imports(project, source, resource, imports): def _get_pyname(project, resource, offset): pymodule = project.get_pymodule(resource) + word_range = worder.Worder(pymodule.source_code, True).get_word_range(offset) + if word_range in _lambda_owned_name_ranges(pymodule): + raise exceptions.RefactoringError( + "Inlining lambda-local bindings is not supported." + ) pyname = evaluate.eval_location(pymodule, offset) if isinstance(pyname, pynames.ImportedName): pyname = pyname._get_imported_pyname() return pyname +def _check_lambda_occurrences(finder, pymodule, region=None, writes=True): + owned = _lambda_owned_name_ranges(pymodule) + if not owned: + return + for occurrence in finder.find_occurrences(pymodule=pymodule): + if occurrence.is_a_fixed_primary(): + continue + if not writes and occurrence.is_written(): + continue + start, _ = occurrence.get_primary_range() + if region is not None and not region[0] <= start < region[1]: + continue + if occurrence.get_word_range() in owned: + raise exceptions.RefactoringError( + "Cannot inline a variable into lambda-local bindings." + ) + + +def _lambda_owned_name_ranges(pymodule): + """Find lambda-local tokens and attribute lookups that depend on them.""" + ranges = set() + lines = pymodule.lines + words = worder.Worder(pymodule.source_code, True) + + def offset(lineno, column): + # AST columns count UTF-8 bytes; Rope offsets count source characters. + prefix = lines.get_line(lineno).encode("utf-8")[:column] + return lines.get_line_start(lineno) + len(prefix.decode("utf-8")) + + def node_range(node): + return ( + offset(node.lineno, node.col_offset), + offset(node.end_lineno, node.end_col_offset), + ) + + def record(node): + ranges.add(node_range(node)) + + def defaults(node): + return [*node.args.defaults, *filter(None, node.args.kw_defaults)] + + def uses_local_name(node): + if isinstance(node, ast.Name): + return node_range(node) in ranges + # Creating a lambda does not evaluate its body or bind its parameters. + children = ( + defaults(node) + if isinstance(node, ast.Lambda) + else list(ast.iter_child_nodes(node)) + ) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Lambda): + children.append(node.func.body) + return any(uses_local_name(child) for child in children) + + def assigned_names(node): + names = set() + if isinstance(node, ast.NamedExpr): + names.add(node.target.id) + children = ( + defaults(node) + if isinstance(node, ast.Lambda) + else ast.iter_child_nodes(node) + ) + for child in children: + names.update(assigned_names(child)) + return names + + def visit(node, bound): + if isinstance(node, ast.Lambda): + arguments = [*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs] + arguments.extend(arg for arg in (node.args.vararg, node.args.kwarg) if arg) + for arg in arguments: + record(arg) + for default in defaults(node): + visit(default, bound) + local = bound | {arg.arg for arg in arguments} | assigned_names(node.body) + visit(node.body, local) + elif isinstance( + node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp) + ): + # The first iterable is outside the comprehension's local bindings. + visit(node.generators[0].iter, bound) + names = { + child.id + for generator in node.generators + for child in ast.walk(generator.target) + if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Store) + } + local = bound - names + for index, generator in enumerate(node.generators): + visit(generator.target, local) + if index: + visit(generator.iter, local) + for condition in generator.ifs: + visit(condition, local) + values = ( + [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] + ) + for value in values: + visit(value, local) + else: + if isinstance(node, ast.Name) and node.id in bound: + record(node) + for child in ast.iter_child_nodes(node): + visit(child, bound) + if isinstance(node, ast.Attribute) and uses_local_name(node.value): + # The attr token resolves through the receiver, not its spelling. + end = offset(node.end_lineno, node.end_col_offset) + ranges.add(words.get_word_range(end - 1)) + + visit(pymodule.get_ast(), set()) + return ranges + + def _remove_from(project, pyname, source, resource): - pymodule = libutils.get_string_module(project, source, resource) + pymodule = libutils.get_string_module( + project, source if source is not None else resource.read(), resource + ) module_import = importutils.get_module_imports(project, pymodule) - module_import.remove_pyname(pyname) - return module_import.get_changed_source() + if module_import.remove_pyname(pyname): + source = module_import.get_changed_source() + return source diff --git a/rope/refactor/introduce_parameter.py b/rope/refactor/introduce_parameter.py index 93b30d0be..d79e7ece0 100644 --- a/rope/refactor/introduce_parameter.py +++ b/rope/refactor/introduce_parameter.py @@ -1,5 +1,5 @@ import rope.base.change # Use fully qualified names for clarity. -from rope.base import codeanalyze, evaluate, exceptions, worder +from rope.base import codeanalyze, evaluate, exceptions, pyscopes, worder from rope.refactor import functionutils, occurrences, sourceutils @@ -39,13 +39,25 @@ def __init__(self, project, resource, offset): self.resource = resource self.offset = offset self.pymodule = self.project.get_pymodule(self.resource) - scope = self.pymodule.get_scope().get_inner_scope_for_offset(offset) - if scope.get_kind() != "Function": + context = self.pymodule.get_source_binding_index().context_at(offset) + scope = context.environment + self.name, self.pyname = self._get_name_and_pyname() + binding_scope = getattr(self.pyname, "binding_scope", None) + if ( + scope.get_kind() in {"Lambda", "Comprehension"} + and context.annotation_owner is None + and ( + binding_scope is None + or binding_scope.get_kind() + not in {"Lambda", "Comprehension", "TypeParameters"} + ) + ): + scope = context.structural_scope + if not isinstance(scope, pyscopes.FunctionScope): raise exceptions.RefactoringError( "Introduce parameter should be performed inside functions" ) self.pyfunction = scope.pyobject - self.name, self.pyname = self._get_name_and_pyname() if self.pyname is None: raise exceptions.RefactoringError( "Cannot find the definition of <%s>" % self.name @@ -62,6 +74,11 @@ def _get_name_and_pyname(self): ) def get_changes(self, new_parameter): + functionutils._check_signature_parameters(self.pyfunction) + if not getattr(self.pyname, "is_refactoring_identity_safe", True): + raise exceptions.RefactoringError( + "Cannot refactor a name with an unsupported source binding." + ) definition_info = functionutils.DefinitionInfo.read(self.pyfunction) definition_info.args_with_defaults.append((new_parameter, self._get_primary())) collector = codeanalyze.ChangeCollector(self.resource.read()) diff --git a/rope/refactor/localtofield.py b/rope/refactor/localtofield.py index e936d5161..bfa8d1ebc 100644 --- a/rope/refactor/localtofield.py +++ b/rope/refactor/localtofield.py @@ -35,7 +35,16 @@ def _check_redefinition(self, name, function_scope): raise exceptions.RefactoringError("The field %s already exists" % name) def _get_field_name(self, pyfunction, name): - self_name = pyfunction.get_param_names()[0] + if pyfunction.get_kind() == "staticmethod": + raise exceptions.RefactoringError( + "Cannot convert a local variable to a field without a method receiver." + ) + positional = pyfunction.get_positional_param_names() + if not positional: + raise exceptions.RefactoringError( + "Cannot convert a local variable to a field without a method receiver." + ) + self_name = positional[0] new_name = self_name + "." + name return new_name diff --git a/rope/refactor/move.py b/rope/refactor/move.py index f13aca6d1..089c4f63b 100644 --- a/rope/refactor/move.py +++ b/rope/refactor/move.py @@ -7,10 +7,14 @@ from __future__ import annotations +import builtins +import keyword +import re import typing -from typing import List, Optional, Union +from collections.abc import Mapping from rope.base import ( + ast, codeanalyze, evaluate, exceptions, @@ -22,6 +26,7 @@ ) from rope.base.change import ChangeContents, ChangeSet, MoveResource from rope.refactor import functionutils, importutils, occurrences, rename, sourceutils +from rope.refactor.importutils import importinfo from rope.refactor.importutils.module_imports import ( get_first_decorator_or_function_start_line, ) @@ -34,7 +39,7 @@ def create_move(project, resource, offset=None): """A factory for creating Move objects Based on `resource` and `offset`, return one of `MoveModule`, - `MoveGlobal` or `MoveMethod` for performing move refactoring. + `MoveGlobal`, `MoveClass` or `MoveMethod` for performing move refactoring. """ if offset is None: @@ -43,14 +48,16 @@ def create_move(project, resource, offset=None): pyname = evaluate.eval_location(this_pymodule, offset) if pyname is not None: pyobject = pyname.get_object() - if isinstance(pyobject, pyobjects.PyModule) or isinstance( - pyobject, pyobjects.PyPackage - ): + if isinstance(pyobject, (pyobjects.PyModule, pyobjects.PyPackage)): return MoveModule(project, pyobject.get_resource()) if isinstance(pyobject, pyobjects.PyFunction) and isinstance( pyobject.parent, pyobjects.PyClass ): return MoveMethod(project, resource, offset) + if isinstance(pyobject, pyobjects.PyClass) and isinstance( + pyobject.parent, pyobjects.PyClass + ): + return MoveClass(project, resource, offset) if ( isinstance(pyobject, pyobjects.PyDefinedObject) and isinstance(pyobject.parent, pyobjects.PyModule) @@ -69,6 +76,13 @@ class MoveMethod: the body of the old method to call the new method. You can inline the old method to change all of its occurrences. + ``get_class_changes()`` explicitly transfers static and class methods + together to an existing source class, without forwarding methods. It + preserves their declarations and rewrites proven references in the + selected resources. Unresolved references, overrides, destination + collisions and dependencies on the old class reject the whole plan. + Planning never writes resources; callers apply its single ChangeSet. + """ def __init__(self, project, resource, offset): @@ -80,11 +94,1868 @@ def __init__(self, project, resource, offset): if self.pyfunction.get_kind() != "method": raise exceptions.RefactoringError("Only normal methods can be moved.") + @classmethod + def get_class_changes( + cls, + project, + moves, + destination_resource, + destination_offset, + resources=None, + task_handle=taskhandle.DEFAULT_TASK_HANDLE, + *, + match_filter=None, + ): + """Transfer decorated methods to an existing class in one ChangeSet. + + ``moves`` contains ``(resource, offset, new_name)`` triples. The + destination offset identifies a source class. Only native static + and class methods with no dependency on the former receiver are + admitted. References between transferred methods retain ``cls`` only + when it resolves to the exact lexical receiver. Every rewritten + destination is proven in its consumer's native lexical environment. + Private identifiers must retain their effective lexical spelling; + implicit construction and subscription slots are not transferable. + Locally defined bindings retain their native lexical identity without + requiring inference of their values. Global and imported bindings + require resolved values. A private declaration whose effective name + is not closed by the native occurrence search is rejected. + Overrides, ambiguous references and changed global bindings fail + before any resource is written. ``resources`` bounds the reference + search and must include all source and destination resources. + ``match_filter`` can exclude sites independently proven by the caller. + """ + destination_module = project.get_pymodule(destination_resource) + binding = evaluate.eval_location(destination_module, destination_offset) + destination = None if binding is None else binding.get_object() + if not isinstance(destination, pyobjects.PyClass): + raise exceptions.RefactoringError( + "Move destination must be a source class." + ) + if destination.get_module() is not destination_module: + raise exceptions.RefactoringError( + "Destination must identify its declaration." + ) + selected = list(project.get_python_files() if resources is None else resources) + selected = list(dict.fromkeys(selected)) + methods = {} + regions = {} + for resource, offset, new_name in moves: + module = project.get_pymodule(resource) + name = evaluate.eval_location(module, offset) + function = None if name is None else name.get_object() + if ( + not isinstance(function, pyobjects.PyFunction) + or not isinstance(function.parent, pyobjects.PyClass) + or function.get_module() is not module + ): + raise exceptions.RefactoringError( + "Move source must identify a method declaration." + ) + if ( + not new_name.isidentifier() + or keyword.iskeyword(new_name) + or new_name in destination.get_attributes() + or new_name in (item[1] for item in methods.values()) + or function in methods + ): + raise exceptions.RefactoringError( + "Duplicate or occupied method destination." + ) + cls._check_class_method(function) + cls._check_class_move_definition_time(function, destination) + if ( + cls._class_move_effective_name(function.get_name(), function.parent) + != function.get_name() + or cls._class_move_effective_name(new_name, destination) != new_name + ): + raise exceptions.RefactoringError( + "Private method declaration has references outside the native name search." + ) + cls._check_class_move_lexical(function, destination) + cls._check_class_move_slots(function.parent, (function,)) + methods[function] = (name, new_name) + regions[function] = cls._class_method_region(function) + if not methods: + raise exceptions.RefactoringError("No methods selected for class transfer.") + cls._check_class_move_construction(destination) + if destination_resource not in selected or any( + function.get_module().get_resource() not in selected for function in methods + ): + raise exceptions.RefactoringError( + "Move resources exclude a source or destination." + ) + destination_path = [] + scope = destination.get_scope() + while scope.get_kind() != "Module": + if scope.get_kind() != "Class": + raise exceptions.RefactoringError( + "Destination class must have a module-owned path." + ) + destination_path.insert(0, scope.pyobject.get_name()) + scope = scope.parent + target = ".".join(destination_path) + placeholder = "__rope_class_move_destination__" + sources = {resource: resource.read() for resource in selected} + if any(placeholder in source for source in sources.values()): + raise exceptions.RefactoringError("Move placeholder conflicts with source.") + edits = {resource: {} for resource in selected} + edges = {function: [] for function in methods} + for function, (name, new_name) in methods.items(): + occurrences.check_opaque_references( + project, name, selected, name=function.get_name() + ) + finder = occurrences.create_finder( + project, + function.get_name(), + name, + imports=False, + in_hierarchy=True, + unsure=lambda occurrence: True, + ) + for resource in selected: + module = project.get_pymodule(resource) + for occurrence in finder.find_occurrences(pymodule=module): + if ( + not occurrence.is_defined() + and match_filter is not None + and not match_filter(occurrence) + ): + continue + actual = occurrence.get_pyname() + if ( + occurrences.unsure_pyname(name, unbound=False) + or occurrences.unsure_pyname(actual, unbound=False) + or not occurrences.same_pyname(name, actual) + ): + raise exceptions.RefactoringError( + f"Unproven reference or override: {resource.path}:{occurrence.lineno}" + ) + if occurrence.is_defined(): + continue + if occurrence.is_written() or occurrence.is_in_import_statement(): + raise exceptions.RefactoringError( + "Cannot transfer a written method reference." + ) + start, end = occurrence.get_primary_range() + enclosing = next( + ( + owner + for owner, (first, last) in regions.items() + if owner.get_module() is module and first <= start < last + ), + None, + ) + if ( + resource == destination_resource + and enclosing is None + and occurrence.lineno <= destination.get_ast().end_lineno + and cls._class_move_runs_at_import(module, start) + ): + raise exceptions.RefactoringError( + "Transfer cannot prove destination exists before reference." + ) + words = worder.Worder(module.source_code) + prefix, _, _ = words.get_splitted_primary_before(end) + environment = ( + module.get_source_binding_index().context_at(start).environment + ) + receiver = ( + evaluate.eval_str(environment, prefix) if prefix else None + ) + if prefix and occurrences.unsure_pyname(receiver, unbound=False): + raise exceptions.RefactoringError( + "Method receiver has no resolved native binding." + ) + parameter = None + if enclosing is not None: + edges[enclosing].append((start, end, function)) + if enclosing.get_kind() == "classmethod": + parameter = enclosing.get_scope().get_names()[ + enclosing.get_param_names()[0] + ] + if ( + parameter is not None + and not occurrences.unsure_pyname(parameter, unbound=False) + and not occurrences.unsure_pyname(receiver, unbound=False) + and occurrences.same_pyname(parameter, receiver) + ): + start, end = occurrence.get_word_range() + replacement = new_name + else: + if prefix and not all( + part.isidentifier() for part in prefix.split(".") + ): + raise exceptions.RefactoringError( + "Effectful method receiver cannot be removed." + ) + if "." in prefix and ( + occurrences.unsure_pyname(receiver, unbound=False) + or not isinstance( + receiver.get_object(), + (pyobjects.AbstractClass, pyobjects.AbstractModule), + ) + ): + raise exceptions.RefactoringError( + "Unproven method receiver evaluation." + ) + replacement = f"{placeholder}.{new_name}" + edits[resource][start, end] = replacement + imports = [] + moved_sources = [] + for function, (_, new_name) in methods.items(): + module = function.get_module() + resource = module.get_resource() + start, end = regions[function] + cls._check_class_receiver(function, methods) + collector = codeanalyze.ChangeCollector(sources[resource][start:end]) + words = worder.Worder(module.source_code) + first = words.find_function_offset( + module.lines.get_line_start(function.get_ast().lineno) + ) + last = first + len(function.get_name()) + collector.add_change(first - start, last - start, new_name) + for (first, last), replacement in edits[resource].items(): + if start <= first and last <= end: + collector.add_change(first - start, last - start, replacement) + moving = sourceutils.fix_indentation( + collector.get_changed() or sources[resource][start:end], 0 + ) + if resource != destination_resource: + moving, used_imports = moving_code_with_imports( + project, resource, moving + ) + imports.extend(used_imports) + moved_sources.append(moving.rstrip()) + planned = {} + for resource, source in sources.items(): + collector = codeanalyze.ChangeCollector(source) + removed = [ + function + for function in methods + if function.get_module().get_resource() == resource + ] + for function in removed: + first, last = regions[function] + parent = function.parent + siblings = [item for item in methods if item.parent is parent] + empty = all( + node in [item.get_ast() for item in siblings] + for node in parent.get_ast().body + ) + replacement = "" + if empty and function is siblings[0]: + replacement = sourceutils.fix_indentation( + "pass\n", + sourceutils.get_indents( + function.get_module().lines, function.get_ast().lineno + ), + ) + collector.add_change(first, last, replacement) + for (first, last), replacement in edits[resource].items(): + if not any( + regions[item][0] <= first < regions[item][1] for item in removed + ): + collector.add_change(first, last, replacement) + result = collector.get_changed() + if result is not None: + planned[resource] = result + module = libutils.get_string_module( + project, + planned.get(destination_resource, sources[destination_resource]), + destination_resource, + ) + scope = module.get_scope() + for part in destination_path: + matches = [ + child + for child in scope.get_scopes() + if child.get_kind() == "Class" and child.pyobject.get_name() == part + ] + if len(matches) != 1: + raise exceptions.RefactoringError( + "Destination class is no longer unique." + ) + scope = matches[0] + result = sourceutils.add_methods(module, scope, moved_sources) + module = libutils.get_string_module(project, result, destination_resource) + cls._check_class_move_imports(project, module, imports) + result = _add_imports_to_module( + importutils.ImportTools(project), module, imports + ) + planned[destination_resource] = result + for resource, source in list(planned.items()): + if placeholder in source: + module = libutils.get_string_module(project, source, resource) + if resource == destination_resource: + imported = target + else: + source, imported = importutils.add_import( + project, + module, + libutils.modname(destination_resource), + destination_path[0], + ) + imported = ".".join((imported, *destination_path[1:])) + imported_module = libutils.get_string_module(project, source, resource) + expected = destination + if resource == destination_resource: + expected = imported_module + for part in destination_path: + expected = expected[part].get_object() + index = imported_module.get_source_binding_index() + finder = occurrences.Finder(project, placeholder) + for occurrence in finder.find_occurrences(pymodule=imported_module): + environment = index.context_at(occurrence.offset).environment + imported_name = evaluate.eval_str(environment, imported) + if ( + occurrences.unsure_pyname(imported_name, unbound=False) + or imported_name.get_object() is not expected + ): + raise exceptions.RefactoringError( + "Destination import conflicts with a consumer binding." + ) + planned[resource] = source.replace(placeholder, imported) + cls._check_class_move_globals( + project, + destination_resource, + planned[destination_resource], + destination_path, + methods, + edges, + ) + changes = ChangeSet(f"Moving methods to class <{target}>") + jobs = task_handle.create_jobset("Collecting Changes", len(planned)) + for resource, source in planned.items(): + jobs.started_job(resource.path) + if source != sources[resource]: + changes.add_change(ChangeContents(resource, source)) + jobs.finished_job() + return changes + + @classmethod + def get_instance_class_changes( + cls, + project, + moves, + destination_resource, + destination_offset, + *, + receiver_rewrites: Mapping[str, str], + resources=None, + task_handle=taskhandle.DEFAULT_TASK_HANDLE, + resolve_binding=evaluate.eval_node, + ): + """Transfer one identity-decorated instance handler to a source class. + + ``receiver_rewrites`` maps an old ``self.member`` access to a dotted + member path on the new receiver, for example ``{"queue": + "queue_service.queue"}``. The old declaration is removed; no forwarding + method is emitted. Only a single source method decorated by a statically + proven identity decorator is admitted. Every old receiver access and + every method reference must be accounted for before a change set exists. + + This narrow operation is intended for application handlers whose + decorator attaches metadata to, and returns, the original function. + Unknown decorators, receiver expressions, references, or import + identities fail closed. + + ``resources`` follows rope's usual contract: it is the complete set of + files the refactoring analyzes and changes, and defaults to every + Python file of the project. Because no forwarding method is left + behind, a caller that passes a subset asserts that no file outside it + references the method; references are only searched in the selected + files. ``new_name`` renames the transferred method and every + ``self.`` access inside it. + """ + if len(moves) != 1: + raise exceptions.RefactoringError( + "Instance transfer admits exactly one method per change set." + ) + destination_module = project.get_pymodule(destination_resource) + destination_binding = evaluate.eval_location( + destination_module, destination_offset + ) + destination = ( + None if destination_binding is None else destination_binding.get_object() + ) + if not isinstance(destination, pyobjects.PyClass): + raise exceptions.RefactoringError( + "Move destination must be a source class." + ) + if destination.get_module() is not destination_module: + raise exceptions.RefactoringError( + "Destination must identify its declaration." + ) + + resource, offset, new_name = moves[0] + module = project.get_pymodule(resource) + name = evaluate.eval_location(module, offset) + function = None if name is None else name.get_object() + if ( + not isinstance(function, pyobjects.PyFunction) + or not isinstance(function.parent, pyobjects.PyClass) + or function.get_module() is not module + or function.get_kind() != "method" + ): + raise exceptions.RefactoringError( + "Instance transfer must identify an owned instance method." + ) + source_class = function.parent + if source_class is destination: + raise exceptions.RefactoringError( + "Instance transfer source and destination are identical." + ) + if ( + not new_name.isidentifier() + or keyword.iskeyword(new_name) + or new_name in destination.get_attributes() + ): + raise exceptions.RefactoringError( + "Duplicate or occupied method destination." + ) + cls._check_instance_private_identifiers( + function, source_class, destination, new_name + ) + cls._check_identity_decorator(function, resolve_binding=resolve_binding) + + selected = list(project.get_python_files() if resources is None else resources) + selected = list(dict.fromkeys(selected)) + source_resource = function.get_module().get_resource() + if source_resource not in selected or destination_resource not in selected: + raise exceptions.RefactoringError( + "Move resources exclude a source or destination." + ) + + region = cls._class_method_region(function) + source = resource.read() + moved = cls._rewrite_instance_receivers( + function, + source, + region, + receiver_rewrites, + destination, + new_name, + ) + occurrences.check_opaque_references( + project, + name, + selected, + name=function.get_name(), + ) + cls._check_instance_references(project, function, selected, region, new_name) + + moved = sourceutils.fix_indentation(moved, 0) + if resource == destination_resource: + # Same-module transfer: local dependencies stay in scope; importing + # them would make the module import itself. + imports = [] + else: + moved, imports = moving_code_with_imports(project, resource, moved) + planned = {item: item.read() for item in selected} + collector = codeanalyze.ChangeCollector(source) + replacement = "" + body = source_class.get_ast().body + if len(body) == 1 and function.get_ast() is body[0]: + replacement = sourceutils.fix_indentation( + "pass\n", + sourceutils.get_indents( + function.get_module().lines, function.get_ast().lineno + ), + ) + collector.add_change(*region, replacement) + changed = collector.get_changed() + if changed is None: + raise exceptions.RefactoringError( + "Instance transfer produced no source change." + ) + planned[resource] = changed + + destination_source = planned[destination_resource] + candidate = libutils.get_string_module( + project, destination_source, destination_resource + ) + destination_scope = cls._planned_class_scope(candidate, destination) + destination_start, destination_end = sourceutils.get_body_region( + destination_scope.pyobject + ) + body_source = destination_source[destination_start:destination_end] + if body_source.strip() != "pass": + destination_start = destination_end + prefix = "\n\n" + else: + prefix = "\n" + indents = sourceutils.get_indents( + destination_scope.pyobject.get_module().lines, + destination_scope.pyobject.get_ast().lineno, + ) + sourceutils.get_indent(project) + inserted = sourceutils.fix_indentation(moved.rstrip(), indents) + collector = codeanalyze.ChangeCollector(destination_source) + collector.add_change( + destination_start, + destination_end, + prefix + inserted + "\n", + ) + planned[destination_resource] = collector.get_changed() + destination_module_candidate = libutils.get_string_module( + project, planned[destination_resource], destination_resource + ) + cls._check_class_move_imports( + project, + destination_module_candidate, + imports, + ) + planned[destination_resource] = _add_imports_to_module( + importutils.ImportTools(project), destination_module_candidate, imports + ) + destination_module_candidate = libutils.get_string_module( + project, planned[destination_resource], destination_resource + ) + cls._check_instance_destination( + function, + destination_module_candidate, + destination, + new_name, + resolve_binding=resolve_binding, + ) + + changes = ChangeSet( + f"Moving instance method <{function.get_name()}> " + f"to class <{destination.get_name()}>" + ) + jobs = task_handle.create_jobset("Collecting Changes", len(planned)) + for selected_resource, contents in planned.items(): + jobs.started_job(selected_resource.path) + if contents != selected_resource.read(): + changes.add_change(ChangeContents(selected_resource, contents)) + jobs.finished_job() + return changes + + @classmethod + def _check_identity_decorator(cls, function, *, resolve_binding=evaluate.eval_node): + decorators = function.decorators + if len(decorators) != 1 or not isinstance(decorators[0], ast.Call): + raise exceptions.RefactoringError( + "Instance transfer requires one statically proven identity decorator." + ) + scope = function.parent.get_scope() + cls._require_declaration_expression( + decorators[0].func, + "handler decorator", + scope, + resolve_binding=resolve_binding, + ) + for argument in ( + *decorators[0].args, + *(kw.value for kw in decorators[0].keywords), + ): + cls._require_declaration_expression( + argument, "handler decorator", scope, resolve_binding=resolve_binding + ) + decorator = evaluate.eval_node(function.parent.get_scope(), decorators[0]) + if occurrences.unsure_pyname(decorator, unbound=False): + raise exceptions.RefactoringError( + "Instance decorator has no proven native identity." + ) + decorator_function = decorator.get_object() + if not isinstance(decorator_function, pyobjects.PyFunction): + raise exceptions.RefactoringError( + "Instance decorator does not resolve to a source function." + ) + factory = decorator_function.parent + if not isinstance(factory, pyobjects.PyFunction): + raise exceptions.RefactoringError( + "Instance decorator factory has no source function." + ) + cls._check_identity_decorator_factory(factory, decorator_function) + cls._check_identity_decorator_body(decorator_function) + + @staticmethod + def _check_identity_decorator_factory(factory, decorator_function): + name = decorator_function.get_name() + direct_functions = [ + node for node in factory.get_ast().body if isinstance(node, ast.FunctionDef) + ] + if len(direct_functions) != 1 or direct_functions[0].name != name: + raise exceptions.RefactoringError( + "Decorator factory does not declare one native handler decorator." + ) + returns = [ + node + for node in MoveMethod._walk_scope(factory.get_ast()) + if isinstance(node, ast.Return) + ] + if not returns or any( + not isinstance(node.value, ast.Name) or node.value.id != name + for node in returns + ): + raise exceptions.RefactoringError( + "Decorator factory does not return its proven declaration decorator." + ) + if any( + isinstance(node, ast.Name) + and node.id == name + and isinstance(node.ctx, ast.Store) + for node in MoveMethod._walk_scope(factory.get_ast()) + ): + raise exceptions.RefactoringError( + "Decorator factory rebinds its declaration decorator." + ) + + @staticmethod + def _check_identity_decorator_body(decorator_function): + parameters = decorator_function.get_param_names() + if not parameters: + raise exceptions.RefactoringError( + "Identity decorator has no function parameter." + ) + parameter = parameters[0] + function_ast = decorator_function.get_ast() + if isinstance(function_ast, ast.AsyncFunctionDef) or any( + isinstance(node, (ast.Yield, ast.YieldFrom)) + for node in MoveMethod._walk_scope(function_ast) + ): + raise exceptions.RefactoringError( + "Identity decorator cannot be asynchronous or a generator." + ) + returns = [ + node + for node in MoveMethod._walk_scope(function_ast) + if isinstance(node, ast.Return) + ] + if not returns or any( + not isinstance(node.value, ast.Name) or node.value.id != parameter + for node in returns + ): + raise exceptions.RefactoringError( + "Decorator does not return the original method on every path." + ) + allowed = set() + metadata_names = set() + parents = { + id(child): parent + for parent in MoveMethod._walk_scope(function_ast) + for child in ast.iter_child_nodes(parent) + } + for node in MoveMethod._walk_scope(function_ast): + if not isinstance(node, ast.Call) or not node.args: + continue + if ( + isinstance(node.func, ast.Name) + and node.func.id in {"hasattr", "setattr"} + and isinstance(node.args[0], ast.Name) + and node.args[0].id == parameter + ): + expected = 2 if node.func.id == "hasattr" else 3 + resolved = evaluate.eval_node(decorator_function.get_scope(), node.func) + builtin = getattr(resolved.get_object(), "builtin", None) + if ( + len(node.args) != expected + or node.keywords + or builtin is not getattr(builtins, node.func.id) + ): + raise exceptions.RefactoringError( + "Identity decorator metadata operation is not proven." + ) + value = MoveMethod._literal_source_string( + decorator_function.get_scope(), node.args[1] + ) + if ( + not isinstance(value, str) + or not value.startswith("_") + or value.startswith("__") + ): + raise exceptions.RefactoringError( + "Identity decorator must attach one private metadata attribute." + ) + metadata_names.add(value) + allowed.add(id(node.args[0])) + if len(metadata_names) != 1: + raise exceptions.RefactoringError( + "Identity decorator metadata attribute is not consistent." + ) + MoveMethod._check_identity_decorator_flow( + decorator_function, parameter, next(iter(metadata_names)) + ) + for node in MoveMethod._walk_scope(function_ast): + if not isinstance(node, ast.Name) or node.id != parameter: + continue + parent = parents.get(id(node)) + if id(node) in allowed: + continue + if isinstance(parent, ast.Return) and parent.value is node: + continue + if isinstance(parent, ast.Call) and parent.args and parent.args[0] is node: + continue + raise exceptions.RefactoringError( + "Identity decorator uses its method parameter outside a proven metadata operation." + ) + + @staticmethod + def _check_identity_decorator_flow(decorator_function, parameter, metadata_name): + statements = list(decorator_function.get_ast().body) + if ( + statements + and isinstance(statements[0], ast.Expr) + and isinstance(statements[0].value, ast.Constant) + and isinstance(statements[0].value.value, str) + ): + statements.pop(0) + if not statements or not isinstance(statements[-1], ast.Return): + raise exceptions.RefactoringError( + "Identity decorator has a path without an explicit return." + ) + result = statements.pop() + if not isinstance(result.value, ast.Name) or result.value.id != parameter: + raise exceptions.RefactoringError( + "Identity decorator does not return the original method." + ) + for statement in statements: + if isinstance(statement, ast.Assign): + if len(statement.targets) != 1 or not isinstance( + statement.targets[0], ast.Name + ): + raise exceptions.RefactoringError( + "Identity decorator writes outside a local declaration." + ) + value = statement.value + if isinstance(value, ast.Call): + constructor = evaluate.eval_node( + decorator_function.get_scope(), value.func + ) + created = constructor.get_object() + if ( + not isinstance(created, pyobjects.PyClass) + or created.get_name() != "DecoratorConfig" + or created.get_module().get_name() + != "flext_core._models.handler" + or not isinstance(created.parent, pyobjects.PyClass) + or created.parent.get_name() != "FlextModelsHandler" + ): + raise exceptions.RefactoringError( + "Identity decorator metadata constructor lacks its canonical source identity." + ) + elif not isinstance(value, ast.Constant): + raise exceptions.RefactoringError( + "Identity decorator has an unproven local computation." + ) + continue + if isinstance(statement, ast.If): + test = statement.test + if ( + statement.orelse + or not isinstance(test, ast.Call) + or not isinstance(test.func, ast.Name) + or test.func.id != "hasattr" + or len(test.args) != 2 + or not isinstance(test.args[0], ast.Name) + or test.args[0].id != parameter + or MoveMethod._literal_source_string( + decorator_function.get_scope(), test.args[1] + ) + != metadata_name + or not statement.body + or not isinstance(statement.body[-1], ast.Raise) + ): + raise exceptions.RefactoringError( + "Identity decorator guard is not a proven metadata check." + ) + raised = statement.body[-1].exc + if not isinstance(raised, ast.Call): + raise exceptions.RefactoringError( + "Identity decorator guard has an unproven rejection." + ) + error = evaluate.eval_node(decorator_function.get_scope(), raised.func) + if getattr(error.get_object(), "builtin", None) is not ValueError: + raise exceptions.RefactoringError( + "Identity decorator guard does not raise a builtin validation error." + ) + if any( + not isinstance(local, ast.Assign) + or len(local.targets) != 1 + or not isinstance(local.targets[0], ast.Name) + or not isinstance(local.value, ast.Constant) + for local in statement.body[:-1] + ): + raise exceptions.RefactoringError( + "Identity decorator guard contains an unproven effect." + ) + continue + if isinstance(statement, ast.Expr) and isinstance( + statement.value, ast.Call + ): + call = statement.value + if ( + not isinstance(call.func, ast.Name) + or call.func.id != "setattr" + or len(call.args) != 3 + or call.keywords + or not isinstance(call.args[0], ast.Name) + or call.args[0].id != parameter + or MoveMethod._literal_source_string( + decorator_function.get_scope(), call.args[1] + ) + != metadata_name + or not isinstance(call.args[2], ast.Name) + ): + raise exceptions.RefactoringError( + "Identity decorator metadata write is not proven." + ) + setter = evaluate.eval_node(decorator_function.get_scope(), call.func) + if ( + getattr(setter.get_object(), "builtin", None) + is not builtins.setattr + ): + raise exceptions.RefactoringError( + "Identity decorator metadata write is not builtin setattr." + ) + continue + raise exceptions.RefactoringError( + "Identity decorator contains an unproven control-flow effect." + ) + + @staticmethod + def _literal_source_string(scope, node): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + binding = evaluate.eval_node(scope, node) + assignments = getattr(binding, "assignments", ()) + if len(assignments) != 1: + return None + value = assignments[0].ast_node + return ( + value.value + if isinstance(value, ast.Constant) and isinstance(value.value, str) + else None + ) + + @classmethod + def _check_instance_private_identifiers( + cls, function, source, destination, new_name + ): + identifiers = [] + for node in ast.walk(function.get_ast()): + if isinstance(node, ast.Name): + identifiers.append(node.id) + elif isinstance(node, ast.Attribute): + identifiers.append(node.attr) + elif ( + isinstance(node, ast.arg) + or isinstance(node, ast.keyword) + and node.arg is not None + ): + identifiers.append(node.arg) + elif isinstance(node, ast.alias): + identifiers.extend(name for name in (node.name, node.asname) if name) + elif ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + or isinstance(node, ast.ExceptHandler) + and node.name is not None + ): + identifiers.append(node.name) + elif type(node).__name__ in {"TypeVar", "ParamSpec", "TypeVarTuple"}: + name = getattr(node, "name", None) + if name is not None: + identifiers.append(name) + + source_name = function.get_name() + private = any( + name.startswith("__") and not name.endswith("__") + for name in (source_name, new_name) + ) + if private and cls._class_move_effective_name( + source_name, source + ) != cls._class_move_effective_name(new_name, destination): + raise exceptions.RefactoringError( + f"Private method name changes under class transfer: {source_name}" + ) + for identifier in identifiers: + if cls._class_move_effective_name( + identifier, source + ) != cls._class_move_effective_name(identifier, destination): + raise exceptions.RefactoringError( + f"Private identifier changes under class transfer: {identifier}" + ) + + @staticmethod + def _walk_scope(root): + pending = list(root.body) + while pending: + node = pending.pop() + yield node + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + pending.extend(ast.iter_child_nodes(node)) + + @classmethod + def _rewrite_instance_receivers( + cls, function, source, region, receiver_rewrites, destination, new_name + ): + parameters = function.get_param_names() + if not parameters: + raise exceptions.RefactoringError("Instance method has no receiver.") + receiver = parameters[0] + positions = codeanalyze.ASTLinesAdapter(source) + collector = codeanalyze.ChangeCollector(source[region[0] : region[1]]) + function_ast = function.get_ast() + nodes = tuple(ast.walk(function_ast)) + if any( + isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef) + ) + for node in nodes + if node is not function_ast + ): + raise exceptions.RefactoringError( + "Instance transfer does not admit nested scopes." + ) + if any( + isinstance(node, ast.Name) and node.id in {"super", "__class__"} + for node in nodes + ): + raise exceptions.RefactoringError( + "Instance transfer depends on its original class closure." + ) + parents = { + id(child): parent + for parent in nodes + for child in ast.iter_child_nodes(parent) + } + used = set() + for node in nodes: + if not isinstance(node, ast.Name) or node.id != receiver: + continue + parent = parents.get(id(node)) + if not isinstance(parent, ast.Attribute) or parent.value is not node: + raise exceptions.RefactoringError( + "Instance receiver escapes a declared member access." + ) + if not isinstance(parent.ctx, ast.Load): + raise exceptions.RefactoringError( + "Instance receiver member is not read-only." + ) + member = parent.attr + if member == function.get_name(): + replacement = f"{receiver}.{new_name}" + else: + suffix = receiver_rewrites.get(member) + if suffix is None: + raise exceptions.RefactoringError( + f"Instance receiver member has no declared port mapping: {member}" + ) + path = suffix.split(".") + if not path or any( + not part.isidentifier() + or keyword.iskeyword(part) + or (part.startswith("__") and not part.endswith("__")) + for part in path + ): + raise exceptions.RefactoringError( + f"Instance receiver mapping is not a safe member path: {suffix}" + ) + if path[0] not in destination.get_attributes(): + raise exceptions.RefactoringError( + f"Instance receiver port is absent from destination: {path[0]}" + ) + replacement = f"{receiver}.{suffix}" + used.add(member) + start, _ = positions[node] + _, end = positions[parent] + collector.add_change(start - region[0], end - region[0], replacement) + if used != set(receiver_rewrites): + raise exceptions.RefactoringError( + "Instance receiver mapping includes an unused member." + ) + if new_name != function.get_name(): + # The transferred declaration takes the requested name. + header, _ = positions[function_ast] + match = re.compile( + r"(?:async\s+)?def\s+(%s)\b" % re.escape(function.get_name()) + ).match(source, header) + if match is None: + raise exceptions.RefactoringError( + "Instance transfer cannot locate its method declaration name." + ) + collector.add_change( + match.start(1) - region[0], match.end(1) - region[0], new_name + ) + changed = collector.get_changed() + return source[region[0] : region[1]] if changed is None else changed + + @staticmethod + def _check_instance_references(project, function, resources, region, new_name): + binding = function.parent.get_attributes().get(function.get_name()) + if binding is None or binding.get_object() is not function: + raise exceptions.RefactoringError( + "Instance transfer source binding is not uniquely owned." + ) + finder = occurrences.create_finder( + project, + function.get_name(), + binding, + imports=False, + in_hierarchy=True, + keywords=False, + ) + for resource in resources: + module = project.get_pymodule(resource) + for occurrence in finder.find_occurrences(pymodule=module): + if occurrence.is_defined(): + continue + start, _ = occurrence.get_primary_range() + if ( + resource == function.get_module().get_resource() + and region[0] <= start < region[1] + ): + continue + raise exceptions.RefactoringError( + f"Instance transfer has an external method reference: {resource.path}:{occurrence.lineno}" + ) + + @classmethod + def _planned_class_scope(cls, module, pyclass): + """Return the scope of `pyclass` re-read from a planned module.""" + scope = module.get_scope() + for part in cls._class_path(pyclass): + matches = [ + child + for child in scope.get_scopes() + if child.get_kind() == "Class" and child.pyobject.get_name() == part + ] + if len(matches) != 1: + raise exceptions.RefactoringError( + "Destination class is no longer unique." + ) + scope = matches[0] + return scope + + @classmethod + def _check_instance_destination( + cls, function, module, destination, name, *, resolve_binding=evaluate.eval_node + ): + destination_scope = cls._planned_class_scope(module, destination) + destination_class = destination_scope.pyobject + binding = destination_class.get_attributes().get(name) + moved = None if binding is None else binding.get_object() + if not isinstance(moved, pyobjects.PyFunction) or moved.get_kind() != "method": + raise exceptions.RefactoringError( + "Transferred declaration is not an instance method." + ) + cls._check_identity_decorator(moved, resolve_binding=resolve_binding) + source_scope = function.parent.get_scope() + if module.get_resource() == function.get_module().get_resource() and ( + cls._is_module_owned(function.parent) + ): + # Same-module transfer: compare both readings in the planned source. + source_scope = cls._planned_class_scope(module, function.parent) + cls._check_instance_signature_bindings( + function, + moved, + source_scope, + destination_class, + resolve_binding=resolve_binding, + ) + if len(moved.decorators) != len(function.decorators): + raise exceptions.RefactoringError( + "Transferred handler decorator declaration changed." + ) + if ast.dump(function.decorators[0]) != ast.dump(moved.decorators[0]): + raise exceptions.RefactoringError( + "Transferred handler decorator arguments changed." + ) + before = resolve_binding(source_scope, function.decorators[0].func) + after = resolve_binding(destination_class.get_scope(), moved.decorators[0].func) + if ( + occurrences.unsure_pyname(before, unbound=False) + or occurrences.unsure_pyname(after, unbound=False) + or not occurrences.same_pyname(before, after) + ): + raise exceptions.RefactoringError( + "Transferred handler decorator changed its native identity." + ) + cls._check_expression_bindings( + source_scope, + destination_class.get_scope(), + function.decorators[0], + moved.decorators[0], + "Transferred handler decorator binding changed.", + resolve_binding=resolve_binding, + ) + + @classmethod + def _check_instance_signature_bindings( + cls, + function, + moved, + source_scope, + destination_class, + *, + resolve_binding=evaluate.eval_node, + ): + source_future = cls._future_annotations(function.get_module()) + destination_future = cls._future_annotations(destination_class.get_module()) + source_annotations = cls._signature_annotations(function.get_ast()) + destination_annotations = cls._signature_annotations(moved.get_ast()) + if ( + source_annotations or destination_annotations + ) and source_future != destination_future: + raise exceptions.RefactoringError( + "Method annotation future policy changes during transfer." + ) + for expression in cls._signature_defaults(function.get_ast()): + cls._require_declaration_expression( + expression, + "method default", + function.parent.get_scope(), + resolve_binding=resolve_binding, + ) + if not source_future: + for expression in source_annotations: + cls._require_declaration_expression( + expression, + "method annotation", + function.parent.get_scope(), + resolve_binding=resolve_binding, + ) + source_expressions = cls._signature_expressions(function.get_ast()) + destination_expressions = cls._signature_expressions(moved.get_ast()) + if len(source_expressions) != len(destination_expressions) or any( + ast.dump(source) != ast.dump(target) + for source, target in zip(source_expressions, destination_expressions) + ): + raise exceptions.RefactoringError( + "Transferred method signature expressions changed." + ) + for source_expression, target_expression in zip( + source_expressions, destination_expressions + ): + cls._check_expression_bindings( + source_scope, + destination_class.get_scope(), + source_expression, + target_expression, + "Transferred method signature binding changed.", + resolve_binding=resolve_binding, + ) + + @staticmethod + def _check_expression_bindings( + source_scope, + destination_scope, + source, + destination, + message, + *, + resolve_binding=evaluate.eval_node, + ): + source_bindings = [ + node + for node in ast.walk(source) + if isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + or isinstance(node, ast.Attribute) + and isinstance(node.ctx, ast.Load) + ] + destination_bindings = [ + node + for node in ast.walk(destination) + if isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + or isinstance(node, ast.Attribute) + and isinstance(node.ctx, ast.Load) + ] + if len(source_bindings) != len(destination_bindings): + raise exceptions.RefactoringError(message) + for source_node, destination_node in zip(source_bindings, destination_bindings): + source_binding = resolve_binding(source_scope, source_node) + destination_binding = resolve_binding(destination_scope, destination_node) + if ( + type(source_node) is not type(destination_node) + or ast.dump(source_node) != ast.dump(destination_node) + or occurrences.unsure_pyname(source_binding, unbound=False) + or occurrences.unsure_pyname(destination_binding, unbound=False) + or not occurrences.same_pyname(source_binding, destination_binding) + ): + raise exceptions.RefactoringError(message) + + @staticmethod + def _signature_expressions(function_ast): + return [ + *MoveMethod._signature_defaults(function_ast), + *MoveMethod._signature_annotations(function_ast), + ] + + @staticmethod + def _signature_defaults(function_ast): + arguments = function_ast.args + expressions = [*arguments.defaults] + expressions.extend( + default for default in arguments.kw_defaults if default is not None + ) + return expressions + + @staticmethod + def _signature_annotations(function_ast): + arguments = function_ast.args + expressions = [] + parameters = [*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs] + parameters.extend( + parameter + for parameter in (arguments.vararg, arguments.kwarg) + if parameter is not None + ) + expressions.extend( + parameter.annotation + for parameter in parameters + if parameter.annotation is not None + ) + if function_ast.returns is not None: + expressions.append(function_ast.returns) + for parameter in getattr(function_ast, "type_params", ()): + expressions.extend( + expression + for expression in ( + getattr(parameter, "bound", None), + getattr(parameter, "default_value", None), + ) + if expression is not None + ) + return expressions + + @staticmethod + def _future_annotations(module): + return any( + isinstance(node, ast.ImportFrom) + and node.module == "__future__" + and any(alias.name == "annotations" for alias in node.names) + for node in module.get_ast().body + ) + + @classmethod + def _require_declaration_expression( + cls, node, owner, scope, *, resolve_binding=evaluate.eval_node + ): + if isinstance(node, (ast.Name, ast.Constant)): + return + if isinstance(node, ast.Attribute): + cls._require_safe_attribute_read( + scope, node, owner, resolve_binding=resolve_binding + ) + cls._require_declaration_expression( + node.value, owner, scope, resolve_binding=resolve_binding + ) + return + if isinstance(node, (ast.Tuple, ast.List, ast.Set)): + for element in node.elts: + cls._require_declaration_expression( + element, owner, scope, resolve_binding=resolve_binding + ) + return + if isinstance(node, ast.Dict): + for key, value in zip(node.keys, node.values): + if key is not None: + cls._require_declaration_expression( + key, owner, scope, resolve_binding=resolve_binding + ) + cls._require_declaration_expression( + value, owner, scope, resolve_binding=resolve_binding + ) + return + raise exceptions.RefactoringError( + f"{owner} contains an expression whose effects are not proven." + ) + + @classmethod + def _require_safe_attribute_read( + cls, scope, node, owner, *, resolve_binding=evaluate.eval_node + ): + resolved = resolve_binding(scope, node) + base_binding = resolve_binding(scope, node.value) + if occurrences.unsure_pyname( + resolved, unbound=False + ) or occurrences.unsure_pyname(base_binding, unbound=False): + raise exceptions.RefactoringError( + f"{owner} attribute binding is not statically resolved." + ) + base = base_binding.get_object() + if isinstance(base, pyobjects.PyModule): + attribute_owner = base + elif isinstance(base, pyobjects.PyClass): + attribute_owner = base + if not cls._native_class_getattribute(base): + raise exceptions.RefactoringError( + f"{owner} class attribute access has no proven native __getattribute__." + ) + else: + attribute_owner = base.get_type() + if not isinstance( + attribute_owner, pyobjects.PyClass + ) or not cls._native_instance_getattribute(attribute_owner): + raise exceptions.RefactoringError( + f"{owner} instance attribute access has no proven native __getattribute__." + ) + binding = attribute_owner.get_attributes().get(node.attr) + if ( + binding is None + or occurrences.unsure_pyname(binding, unbound=False) + or not occurrences.same_pyname(binding, resolved) + ): + raise exceptions.RefactoringError( + f"{owner} attribute is not a direct native declaration: {node.attr}" + ) + value = binding.get_object() + if isinstance(attribute_owner, pyobjects.PyModule): + return + if isinstance(value, pyobjects.PyClass): + return + if ( + isinstance(value, pyobjects.PyFunction) + and value.get_kind() == "staticmethod" + ): + return + literal_type = getattr(value.get_type(), "builtin", None) + if literal_type in (str, int, float, bool, type(None)): + return + raise exceptions.RefactoringError( + f"{owner} attribute may invoke descriptor behavior: {node.attr}" + ) + + @staticmethod + def _native_class_getattribute(pyclass): + metaclass = pyclass.get_metaclass() + if metaclass is None: + return False + if getattr(metaclass, "builtin", None) is type: + native = metaclass.get_attributes().get("__getattribute__") + return ( + native is not None + and getattr(native.get_object(), "builtin", None) + is type.__getattribute__ + ) + if isinstance(metaclass, pyobjects.PyClass): + native = metaclass.get_attributes().get("__getattribute__") + return ( + native is not None + and getattr(native.get_object(), "builtin", None) + is type.__getattribute__ + ) + return False + + @staticmethod + def _native_instance_getattribute(pyclass): + binding = pyclass.get_attributes().get("__getattribute__") + return ( + binding is not None + and getattr(binding.get_object(), "builtin", None) + is object.__getattribute__ + ) + + @staticmethod + def _is_module_owned(pyclass): + scope = pyclass.get_scope().parent + while scope.get_kind() == "Class": + scope = scope.parent + return scope.get_kind() == "Module" + + @staticmethod + def _class_path(pyclass): + path = [] + scope = pyclass.get_scope() + while scope.get_kind() != "Module": + if scope.get_kind() != "Class": + raise exceptions.RefactoringError( + "Destination class must have a module-owned path." + ) + path.insert(0, scope.pyobject.get_name()) + scope = scope.parent + return path + + @staticmethod + def _check_class_method(function): + kind = function.get_kind() + if kind not in ("staticmethod", "classmethod") or len(function.decorators) != 1: + raise exceptions.RefactoringError( + "Only native static and class methods can be transferred." + ) + decorator = evaluate.eval_node( + function.parent.get_scope(), function.decorators[0] + ) + expected = staticmethod if kind == "staticmethod" else classmethod + if ( + decorator is None + or getattr(decorator.get_object(), "builtin", None) is not expected + ): + raise exceptions.RefactoringError( + "Method decorator has no native identity." + ) + if kind == "classmethod" and not function.get_param_names(): + raise exceptions.RefactoringError( + "Class method has no class receiver parameter." + ) + + @staticmethod + def _class_move_runs_at_import(module, offset): + """Whether code at `offset` runs while its module executes. + + Module-level code and class bodies run at import; a function body + runs only when called. + """ + scope = module.get_source_binding_index().context_at(offset).structural_scope + while scope is not None: + if scope.get_kind() == "Function": + return False + scope = scope.parent + return True + + @classmethod + def _check_class_move_definition_time(cls, function, destination): + """Require literal defaults and stable definition-time annotations.""" + node = function.get_ast() + for expression in cls._signature_defaults(node): + cls._require_transfer_literal(expression) + if all( + module.get_source_binding_index().policy.mode == "eager" + for module in (function.get_module(), destination.get_module()) + ): + index = function.get_module().get_source_binding_index() + for expression in cls._signature_annotations(node): + if isinstance(expression, ast.Name): + origin = evaluate.eval_node( + index.context_for_node(expression).environment, expression + ) + target = destination.get_scope().lookup(expression.id) + if origin is not None and target is not None: + builtin = getattr(origin.get_object(), "builtin", None) + if ( + isinstance(builtin, type) + and getattr(builtins, expression.id, None) is builtin + and getattr(target.get_object(), "builtin", None) is builtin + ): + continue + cls._require_transfer_literal(expression) + + @staticmethod + def _require_transfer_literal(expression): + try: + ast.literal_eval(expression) + except (ValueError, TypeError, SyntaxError): + raise exceptions.RefactoringError( + "Transfer cannot prove definition-time evaluation." + ) from None + + _DYNAMIC_NAMESPACE_CALLS = frozenset(("globals", "vars", "locals", "exec", "eval")) + _BUILTIN_MODULE_NAMES = frozenset(("builtins", "__builtins__")) + + @classmethod + def _check_class_move_construction(cls, destination): + """Refuse a destination whose class construction static analysis cannot see. + + A metaclass other than `type` can supply or inspect the namespace + (`__prepare__`, `__new__`), and code run at import time can rebind + any name through `globals()`, `vars()`, `exec` or the builtins module. + Either may change what the moved decorator or body resolves to. + """ + metaclass = destination.get_metaclass() + if getattr(metaclass, "builtin", None) is not type: + raise exceptions.RefactoringError( + "Transfer cannot prove destination class construction." + ) + functions = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda) + pending = [destination.get_module().get_ast()] + while pending: + node = pending.pop() + if isinstance(node, ast.Call) and ( + ( + isinstance(node.func, ast.Name) + and node.func.id in cls._DYNAMIC_NAMESPACE_CALLS + ) + or ( + isinstance(node.func, ast.Name) + and node.func.id in ("setattr", "delattr") + and node.args + and isinstance(node.args[0], ast.Name) + and node.args[0].id in cls._BUILTIN_MODULE_NAMES + ) + ): + raise exceptions.RefactoringError( + "Transfer cannot prove destination namespace bindings." + ) + if ( + isinstance(node, (ast.Attribute, ast.Subscript)) + and isinstance(node.value, ast.Name) + and node.value.id in cls._BUILTIN_MODULE_NAMES + and ( + isinstance(node.ctx, (ast.Store, ast.Del)) + or isinstance(node, ast.Attribute) + and node.attr == "__dict__" + ) + ): + raise exceptions.RefactoringError( + "Transfer cannot prove destination namespace bindings." + ) + for child in ast.iter_child_nodes(node): + if isinstance(child, functions): + # Only definition-time parts of a function run at import. + pending.extend( + child.decorator_list if hasattr(child, "decorator_list") else () + ) + pending.extend(child.args.defaults) + pending.extend(d for d in child.args.kw_defaults if d is not None) + continue + pending.append(child) + + @staticmethod + def _class_method_region(function): + module = function.get_module() + return ( + module.lines.get_line_start( + get_first_decorator_or_function_start_line(function.get_ast()) + ), + min( + module.lines.get_line_end(function.get_scope().get_end()) + 1, + len(module.source_code), + ), + ) + + @staticmethod + def _check_class_move_imports(project, module, imports): + context = importinfo.ImportContext(project, module.get_resource().parent) + for info in imports: + imported_module = libutils.get_string_module( + project, info.get_import_statement() + "\n", module.get_resource() + ) + for name in info.get_imported_names(context): + existing = module.get_attributes().get(name) + incoming = imported_module.get_attributes().get(name) + if occurrences.unsure_pyname(incoming, unbound=False) or ( + existing is not None + and ( + occurrences.unsure_pyname(existing, unbound=False) + or not occurrences.same_pyname(existing, incoming) + ) + ): + raise exceptions.RefactoringError( + f"Moved import conflicts with destination: {name}" + ) + + @staticmethod + def _check_class_receiver(function, methods): + nodes = tuple(ast.walk(function.get_ast())) + index = function.get_module().get_source_binding_index() + parameter = None + if function.get_kind() == "classmethod": + parameter = function.get_scope().get_names()[function.get_param_names()[0]] + if occurrences.unsure_pyname(parameter, unbound=False): + raise exceptions.RefactoringError( + "Class receiver has no resolved native binding." + ) + allowed = set() + for node in nodes: + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + context = index.context_for_node(node.value) + name = evaluate.eval_node(context.environment, node.value) + if ( + parameter is not None + and not occurrences.unsure_pyname(parameter, unbound=False) + and not occurrences.unsure_pyname(name, unbound=False) + and occurrences.same_pyname(parameter, name) + ): + value = evaluate.eval_node( + index.context_for_node(node).environment, node + ) + if ( + not occurrences.unsure_pyname(value, unbound=False) + and getattr(value, "source_pyname", value).get_object() + in methods + ): + allowed.add(id(node.value)) + for node in nodes: + if not isinstance(node, ast.Name): + continue + if node.id in ("super", "__class__"): + raise exceptions.RefactoringError( + "Method depends on its original class closure." + ) + if parameter is not None: + context = index.context_for_node(node) + name = evaluate.eval_node(context.environment, node) + if ( + not occurrences.unsure_pyname(parameter, unbound=False) + and not occurrences.unsure_pyname(name, unbound=False) + and occurrences.same_pyname(parameter, name) + and id(node) not in allowed + ): + raise exceptions.RefactoringError( + "Method retains a dependency on its original class." + ) + + @classmethod + def _check_class_move_globals(cls, project, resource, source, path, methods, edges): + module = libutils.get_string_module(project, source, resource) + owner = module + for part in path: + owner = owner[part].get_object() + candidates = { + original: owner[name].get_object() + for original, (_, name) in methods.items() + } + cls._check_class_move_slots(owner, tuple(candidates.values())) + for original, (_, new_name) in methods.items(): + moved = candidates[original] + cls._check_class_method(moved) + moved_start, moved_end = cls._class_method_region(moved) + observed_edges = [] + for target, (_, target_name) in methods.items(): + expected = owner[target_name] + finder = occurrences.create_finder( + project, + target_name, + expected, + imports=False, + unsure=lambda occurrence: True, + ) + for occurrence in finder.find_occurrences(pymodule=module): + first, last = occurrence.get_primary_range() + if occurrence.is_defined() or not moved_start <= first < moved_end: + continue + actual = occurrence.get_pyname() + if ( + occurrences.unsure_pyname(expected, unbound=False) + or occurrences.unsure_pyname(actual, unbound=False) + or not occurrences.same_pyname(expected, actual) + ): + raise exceptions.RefactoringError( + "Transferred callee has no proven native identity." + ) + observed_edges.append((first, last, target)) + original_edges = sorted(edges[original], key=lambda edge: edge[:2]) + observed_edges.sort(key=lambda edge: edge[:2]) + if [item[2] for item in original_edges] != [ + item[2] for item in observed_edges + ]: + raise exceptions.RefactoringError( + "Transferred method changes its native callee edges." + ) + before_positions = codeanalyze.ASTLinesAdapter( + original.get_module().source_code + ) + after_positions = codeanalyze.ASTLinesAdapter(module.source_code) + original_names = [ + node + for node in ast.walk(original.get_ast()) + if isinstance(node, ast.Name) + and not any( + first <= before_positions[node][0] < last + for first, last, _ in original_edges + ) + ] + moved_names = [ + node + for node in ast.walk(moved.get_ast()) + if isinstance(node, ast.Name) + and not any( + first <= after_positions[node][0] < last + for first, last, _ in observed_edges + ) + ] + if len(original_names) != len(moved_names): + raise exceptions.RefactoringError( + "Transferred method changed its native name structure." + ) + before = original.get_module().get_source_binding_index() + after = module.get_source_binding_index() + for old, new in zip(original_names, moved_names): + context = before.context_for_node(old) + old_name = evaluate.eval_node(context.environment, old) + if old_name is None or not getattr( + old_name, "is_refactoring_identity_safe", True + ): + raise exceptions.RefactoringError( + f"Unproven method binding: {old.id}" + ) + if cls._class_move_local_binding( + context, old.id, old_name, original.get_scope() + ): + continue + if occurrences.unsure_pyname(old_name, unbound=False): + raise exceptions.RefactoringError( + f"Unproven method dependency: {old.id}" + ) + new_name = evaluate.eval_node( + after.context_for_node(new).environment, new + ) + if occurrences.unsure_pyname(new_name, unbound=False) or not ( + occurrences.same_pyname(old_name, new_name) + or cls._class_move_same_module_binding( + original, old.id, old_name, new_name + ) + ): + raise exceptions.RefactoringError( + f"Transferred method changes binding: {old.id}" + ) + + @staticmethod + def _class_move_same_module_binding(function, name, old, new): + """Prove a module-level binding is the same one after a same-module move. + + The moved source is re-parsed, so its module bindings are new objects. + They denote the same binding when both are the one module-level + definition of `name` in the same resource, with an identical + declaration statement. A name the method declares `global` or + `nonlocal` stays unproven, because the method itself can rebind it. + """ + if any( + isinstance(node, (ast.Global, ast.Nonlocal)) and name in node.names + for node in ast.walk(function.get_ast()) + ): + return False + old_module, old_line = old.get_definition_location() + new_module, new_line = new.get_definition_location() + if ( + old_module is None + or new_module is None + or old_line is None + or new_line is None + or old_module.get_resource() != new_module.get_resource() + or old_module.get_scope().get_names().get(name) is not old + or new_module.get_scope().get_names().get(name) is not new + or type(old) is not type(new) + or len(getattr(old, "assignments", ())) > 1 + or len(getattr(new, "assignments", ())) > 1 + ): + return False + + def declaration(module, line): + return [ + ast.dump(node) + for node in module.get_ast().body + if node.lineno <= line <= node.end_lineno + ] + + before = declaration(old_module, old_line) + return len(before) == 1 and before == declaration(new_module, new_line) + + @staticmethod + def _class_move_local_binding(context, name, binding, root_scope): + binding = getattr(binding, "source_pyname", binding) + if isinstance(binding, (pynames.ImportedName, pynames.ImportedModule)): + return False + if binding.get_definition_location()[0] is None: + return False + scope = context.structural_scope + scopes = [] + while scope is not None and scope.get_kind() != "Module": + scopes.append(scope) + if scope is root_scope: + break + scope = scope.parent + if not scopes or scopes[-1] is not root_scope: + return False + if ( + root_scope.pyobject.get_module().get_scope().get_names().get(name) + is binding + ): + return False + for scope in (context.environment, *scopes): + if scope.get_kind() != "Class" and scope.get_names().get(name) is binding: + return True + return False + + @staticmethod + def _check_class_move_slots(owner, functions): + # Follow the same selections as evaluate._Call and _Subscript. + if "__new__" in owner: + name = owner["__new__"] + if occurrences.unsure_pyname(name, unbound=False): + raise exceptions.RefactoringError("Construction slot is unresolved.") + if name.get_object() in functions: + raise exceptions.RefactoringError( + "Implicit construction references are not transferable." + ) + metaclass = owner.get_metaclass() + if not isinstance(metaclass, pyobjects.AbstractClass): + raise exceptions.RefactoringError("Subscription metaclass is unresolved.") + if "__getitem__" in metaclass: + name = metaclass["__getitem__"] + elif "__class_getitem__" in owner: + name = owner["__class_getitem__"] + else: + return + if occurrences.unsure_pyname(name, unbound=False): + raise exceptions.RefactoringError("Subscription slot is unresolved.") + called, _ = evaluate.get_callable(name) + if called in functions: + raise exceptions.RefactoringError( + "Implicit subscription references are not transferable." + ) + + @staticmethod + def _class_move_effective_name(name, owner): + prefix = owner.get_name().lstrip("_") + if ( + name.startswith("__") + and not name.endswith("__") + and "." not in name + and prefix + ): + return f"_{prefix}{name}" + return name + + @classmethod + def _check_class_move_lexical(cls, function, destination): + if function.parent.get_name().lstrip("_") == destination.get_name().lstrip("_"): + return + module = function.get_module() + index = module.get_source_binding_index() + positions = codeanalyze.ASTLinesAdapter(module.source_code) + imports = {layout.region: layout for layout in index.import_layouts()} + + def check_name(name): + old = cls._class_move_effective_name(name, function.parent) + new = cls._class_move_effective_name(name, destination) + if old != new: + raise exceptions.RefactoringError( + f"Transferred method changes private identifier: {name} ({old} -> {new})" + ) + + pending = [function.get_scope()] + visited = set() + while pending: + scope = pending.pop() + if id(scope) in visited or scope.get_kind() == "Class": + continue + visited.add(id(scope)) + for name in scope.get_defined_names(): + check_name(name) + pending.extend(scope.get_scopes()) + pending.extend(scope.get_binding_scopes()) + + def check(node): + if isinstance(node, ast.Name): + check_name(node.id) + elif isinstance(node, ast.Attribute): + check_name(node.attr) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + layout = imports[positions[node]] + for name, alias, _region in layout.aliases: + check_name(name) + if alias is not None: + check_name(alias) + if isinstance(node, ast.ClassDef): + # Its declaration/header belongs to the enclosing context; + # its body keeps the nested class's unchanged private prefix. + for header in (*node.decorator_list, *node.bases, *node.keywords): + ast.call_for_nodes(header, check) + return True + return False + + ast.call_for_nodes(function.get_ast(), check) + def get_changes( self, dest_attr: str, - new_name: Optional[str] = None, - resources: Optional[List[resources.File]] = None, + new_name: str | None = None, + resources: list[resources.File] | None = None, task_handle=taskhandle.DEFAULT_TASK_HANDLE, # FIXME: this is unused ): """Return the changes needed for this refactoring @@ -99,7 +1970,8 @@ def get_changes( will be applied to all python files. """ - changes = ChangeSet("Moving method <%s>" % self.method_name) + functionutils._check_signature_parameters(self.pyfunction) + changes = ChangeSet(f"Moving method <{self.method_name}>") if resources is None: resources = self.project.get_python_files() if new_name is None: @@ -144,11 +2016,7 @@ def _get_used_imports(self, import_tools): def _get_changes_made_by_old_class(self, dest_attr, new_name): pymodule = self.pyfunction.get_module() indents = self._get_scope_indents(self.pyfunction) - body = "return self.{}.{}({})\n".format( - dest_attr, - new_name, - self._get_passed_arguments_string(), - ) + body = f"return self.{dest_attr}.{new_name}({self._get_passed_arguments_string()})\n" region = sourceutils.get_body_region(self.pyfunction) return ( pymodule.get_resource(), @@ -167,12 +2035,12 @@ def _get_changes_made_by_new_class(self, dest_attr, new_name): old_pyclass = self.pyfunction.parent if dest_attr not in old_pyclass: raise exceptions.RefactoringError( - "Destination attribute <%s> not found" % dest_attr + f"Destination attribute <{dest_attr}> not found" ) pyclass = old_pyclass[dest_attr].get_object().get_type() if not isinstance(pyclass, pyobjects.PyClass): raise exceptions.RefactoringError( - "Unknown class type for attribute <%s>" % dest_attr + f"Unknown class type for attribute <{dest_attr}>" ) pymodule = pyclass.get_module() resource = pyclass.get_module().get_resource() @@ -188,6 +2056,7 @@ def _get_changes_made_by_new_class(self, dest_attr, new_name): return resource, start, end, body def get_new_method(self, name): + functionutils._check_signature_parameters(self.pyfunction) return "{}\n{}".format( self._get_new_header(name), sourceutils.fix_indentation( @@ -212,7 +2081,7 @@ def _get_self_name(self): return self.pyfunction.get_param_names()[0] def _get_new_header(self, name): - header = "def %s(self" % name + header = f"def {name}(self" if self._is_host_used(): header += ", host" definition_info = functionutils.DefinitionInfo.read(self.pyfunction) @@ -237,6 +2106,295 @@ def _is_host_used(self): return self._get_body("__old_self") != self._get_unchanged_body() +class MoveClass: + """Transfer a nested class and its native references without a forwarding alias.""" + + def __init__(self, project, resource, offset): + self.project = project + self.source = resource + module = project.get_pymodule(resource) + self.pyname = evaluate.eval_location(module, offset) + self.pyclass = None if self.pyname is None else self.pyname.get_object() + if ( + not isinstance(self.pyclass, pyobjects.PyClass) + or self.pyclass.get_module() is not module + ): + raise exceptions.RefactoringError("Move source must identify a class.") + self._path(self.pyclass) + + @staticmethod + def _path(pyclass): + result = [] + scope = pyclass.get_scope() + while scope.get_kind() != "Module": + if scope.get_kind() != "Class": + raise exceptions.RefactoringError("Class has a function-local closure.") + result.insert(0, scope.pyobject.get_name()) + scope = scope.parent + return result + + def get_changes( + self, + dest, + resources=None, + task_handle=taskhandle.DEFAULT_TASK_HANDLE, + *, + destination_offset=None, + match_filter=None, + ): + """Plan a transfer to a module or the source class at destination_offset.""" + if isinstance(dest, str): + dest = self.project.find_module(dest) + if dest is None or not dest.exists() or dest.is_folder(): + raise exceptions.RefactoringError("Move destination must be a module.") + destination_module = self.project.get_pymodule(dest) + destination = destination_module + path = [] + if destination_offset is not None: + binding = evaluate.eval_location(destination_module, destination_offset) + destination = None if binding is None else binding.get_object() + if ( + not isinstance(destination, pyobjects.PyClass) + or destination.get_module() is not destination_module + ): + raise exceptions.RefactoringError( + "Move destination must identify a class." + ) + path = self._path(destination) + name = self.pyclass.get_name() + if ( + not name.isidentifier() + or keyword.iskeyword(name) + or name in destination.get_attributes() + or MoveMethod._class_move_effective_name(name, self.pyclass.parent) != name + ): + raise exceptions.RefactoringError("Occupied or invalid class destination.") + first, last = MoveMethod._class_method_region(self.pyclass) + if ( + dest == self.source + and destination_offset is not None + and first <= destination_offset < last + ): + raise exceptions.RefactoringError("Cannot move a class inside itself.") + self._check_enclosing_bindings(first, last) + selected = list( + dict.fromkeys( + self.project.get_python_files() if resources is None else resources + ) + ) + if self.source not in selected or dest not in selected: + raise exceptions.RefactoringError( + "Move resources exclude source or destination." + ) + placeholder = "__rope_class_move_destination__" + sources = {resource: resource.read() for resource in selected} + if any(placeholder in source for source in sources.values()): + raise exceptions.RefactoringError("Move placeholder conflicts with source.") + occurrences.check_opaque_references( + self.project, self.pyname, selected, name=self.pyclass.get_name() + ) + finder = occurrences.create_finder( + self.project, + self.pyclass.get_name(), + self.pyname, + imports=False, + unsure=lambda occurrence: True, + ) + edits = {resource: {} for resource in selected} + for resource in selected: + module = self.project.get_pymodule(resource) + for occurrence in finder.find_occurrences(pymodule=module): + if ( + not occurrence.is_defined() + and match_filter is not None + and not match_filter(occurrence) + ): + continue + actual = occurrence.get_pyname() + if occurrences.unsure_pyname( + actual, unbound=False + ) or not occurrences.same_pyname(self.pyname, actual): + raise exceptions.RefactoringError("Unproven class reference.") + if occurrence.is_defined(): + continue + if occurrence.is_written() or occurrence.is_in_import_statement(): + raise exceptions.RefactoringError( + "Cannot move a written class reference." + ) + start, end = occurrence.get_primary_range() + if not all( + part.isidentifier() + for part in occurrence.get_primary_text().split(".") + ): + raise exceptions.RefactoringError( + "Class reference has an effectful receiver." + ) + edits[resource][start, end] = placeholder + "." + name + collector = codeanalyze.ChangeCollector(sources[self.source][first:last]) + if path: + index = self.pyclass.get_module().get_source_binding_index() + for base in self.pyclass.get_ast().bases: + for node in ast.walk(base): + if not isinstance(node, (ast.Name, ast.Attribute)): + continue + binding = evaluate.eval_node( + index.context_for_node(node).environment, node + ) + if occurrences.unsure_pyname(binding, unbound=False): + continue + for member, target_binding in destination.get_attributes().items(): + if ( + binding.get_object() is target_binding.get_object() + and isinstance( + binding.get_object(), pyobjects.AbstractClass + ) + ): + start, end = codeanalyze.ASTLinesAdapter( + sources[self.source] + )[node] + collector.add_change(start - first, end - first, member) + break + for (start, end), replacement in edits[self.source].items(): + if first <= start and end <= last: + collector.add_change(start - first, end - first, replacement) + moving = sourceutils.fix_indentation( + collector.get_changed() or sources[self.source][first:last], 0 + ) + imports = [] + if self.source != dest: + moving, imports = moving_code_with_imports( + self.project, self.source, moving + ) + snippet = ( + "\n".join(info.get_import_statement() for info in imports) + + "\n" + + moving + ) + tools = importutils.ImportTools(self.project) + normalized = tools._remove_self_imports( + libutils.get_string_module(self.project, snippet, dest) + ) + module_imports = tools.module_imports(normalized) + imports = [ + statement.import_info + for statement in module_imports.imports + if not statement.import_info.is_empty() + ] + for statement in module_imports.imports: + statement.empty_import() + moving = module_imports.get_changed_source().lstrip("\n") + planned = {} + for resource, source in sources.items(): + collector = codeanalyze.ChangeCollector(source) + if resource == self.source: + parent = self.pyclass.parent + empty = isinstance( + parent, pyobjects.PyClass + ) and parent.get_ast().body == [self.pyclass.get_ast()] + replacement = ( + sourceutils.fix_indentation( + "pass\n", self.pyclass.get_ast().col_offset + ) + if empty + else "" + ) + collector.add_change(first, last, replacement) + for (start, end), replacement in edits[resource].items(): + if resource != self.source or not first <= start < last: + collector.add_change(start, end, replacement) + changed = collector.get_changed() + planned[resource] = source if changed is None else changed + module = libutils.get_string_module(self.project, planned[dest], dest) + if path: + scope = module.get_scope() + for part in path: + scope = next( + child + for child in scope.get_scopes() + if child.pyobject.get_name() == part + ) + result = sourceutils.add_methods( + module, scope, [moving.rstrip()], at_end=True + ) + else: + result = planned[dest].rstrip() + "\n\n" + moving + module = libutils.get_string_module(self.project, result, dest) + MoveMethod._check_class_move_imports(self.project, module, imports) + planned[dest] = _add_imports_to_module( + importutils.ImportTools(self.project), module, imports + ) + for resource, source in list(planned.items()): + if placeholder + "." not in source: + continue + module = libutils.get_string_module(self.project, source, resource) + if resource == dest: + imported = ".".join(path) + elif path: + source, imported = importutils.add_import( + self.project, module, libutils.modname(dest), path[0] + ) + imported = ".".join((imported, *path[1:])) + else: + source, imported = importutils.add_import( + self.project, module, libutils.modname(dest), name + ) + planned[resource] = source.replace(placeholder + "." + name, imported) + continue + checked = libutils.get_string_module(self.project, source, resource) + expected = destination + if resource == dest: + expected = checked + for part in path: + expected = expected[part].get_object() + index = checked.get_source_binding_index() + for occurrence in occurrences.Finder( + self.project, placeholder + ).find_occurrences(pymodule=checked): + environment = index.context_at(occurrence.offset).environment + binding = evaluate.eval_str(environment, imported) if imported else None + if imported and ( + occurrences.unsure_pyname(binding, unbound=False) + or binding.get_object() is not expected + ): + raise exceptions.RefactoringError( + "Destination import conflicts with a consumer binding." + ) + planned[resource] = source.replace( + placeholder + ".", imported + "." if imported else "" + ) + changes = ChangeSet(f"Moving class <{self.pyclass.get_name()}>") + for resource, source in planned.items(): + if source != sources[resource]: + if resource in (self.source, dest): + module = libutils.get_string_module(self.project, source, resource) + source = importutils.ImportTools(self.project).organize_imports( + module, sort=False + ) + changes.add_change(ChangeContents(resource, source)) + return changes + + def _check_enclosing_bindings(self, first, last): + module = self.pyclass.get_module() + index = module.get_source_binding_index() + for node in ast.walk(self.pyclass.get_ast()): + if not isinstance(node, ast.Name) or not isinstance(node.ctx, ast.Load): + continue + context = index.context_for_node(node) + binding = evaluate.eval_node(context.environment, node) + if occurrences.unsure_pyname(binding, unbound=False): + continue + owner, line = binding.get_definition_location() + if owner is module and line is not None: + offset = module.lines.get_line_start(line) + if not first <= offset < last and ( + node.id not in module.get_attributes() + or not occurrences.same_pyname(module[node.id], binding) + ): + raise exceptions.RefactoringError( + "Class depends on an enclosing class binding." + ) + + class MoveGlobal: """For moving global function and classes""" @@ -263,9 +2421,29 @@ def __init__(self, project, resource, offset): ) self.import_tools = self.tools.import_tools - def _import_filter(self, stmt): + def _import_filter(self, stmt, folder=None): + """Select the statements that import the moved name or its source module. + + ``folder`` is the importing module's folder; relative statements are + resolved from it to the module they import. + """ module_name = libutils.modname(self.source) + if ( + isinstance(stmt.import_info, importutils.FromImport) + and stmt.import_info.level + ): + context = importinfo.ImportContext(self.project, folder) + imported = stmt.import_info.get_imported_resource(context) + source = self.source + if source.name == "__init__.py": + source = source.parent + if imported == source: + return True + basename = module_name.rsplit(".", 1)[-1] + return imported == source.parent and any( + basename == name for name, alias in stmt.import_info.names_and_aliases + ) if isinstance(stmt.import_info, importutils.NormalImport): # Affect any statement that imports the source module return any( @@ -311,8 +2489,8 @@ def _is_variable(self, pyname): def get_changes( self, - dest: Optional[Union[str, resources.Resource]], - resources: Optional[List[resources.File]] = None, + dest: str | resources.Resource | None, + resources: list[resources.File] | None = None, task_handle=taskhandle.DEFAULT_TASK_HANDLE, ): """Return the changes needed for this refactoring @@ -350,7 +2528,7 @@ def get_changes( return self._calculate_changes(dest, resources, task_handle) def _calculate_changes(self, dest, resources, task_handle): - changes = ChangeSet("Moving global <%s>" % self.old_name) + changes = ChangeSet(f"Moving global <{self.old_name}>") job_set = task_handle.create_jobset("Collecting Changes", len(resources)) for file_ in resources: job_set.started_job(file_.path) @@ -361,13 +2539,15 @@ def _calculate_changes(self, dest, resources, task_handle): elif self.tools.occurs_in_module(resource=file_): pymodule = self.project.get_pymodule(file_) # Changing occurrences - placeholder = "__rope_renaming_%s_" % self.old_name + placeholder = f"__rope_renaming_{self.old_name}_" source = self.tools.rename_in_module(placeholder, resource=file_) should_import = source is not None # Removing out of date imports pymodule = self.tools.new_pymodule(pymodule, source) source = self.import_tools.organize_imports( - pymodule, sort=False, import_filter=self._import_filter + pymodule, + sort=False, + import_filter=lambda stmt: self._import_filter(stmt, file_.parent), ) # Adding new import if should_import: @@ -377,13 +2557,47 @@ def _calculate_changes(self, dest, resources, task_handle): ) source = source.replace(placeholder, imported) source = self.tools.new_source(pymodule, source) + source = self._retarget_aliased_imports(file_, source, dest) if source != file_.read(): changes.add_change(ChangeContents(file_, source)) + else: + source = file_.read() + changed = self._retarget_aliased_imports(file_, source, dest) + if changed != source: + changes.add_change(ChangeContents(file_, changed)) job_set.finished_job() return changes + def _retarget_aliased_imports(self, resource, source, dest): + pymodule = libutils.get_string_module(self.project, source, resource) + imports = self.import_tools.module_imports(pymodule) + context = importinfo.ImportContext(self.project, resource.parent) + origin = ( + self.source.parent if self.source.name == "__init__.py" else self.source + ) + for stmt in tuple(imports.imports): + info = stmt.import_info + if not isinstance(info, importutils.FromImport): + continue + aliases = [ + (name, alias) + for name, alias in info.names_and_aliases + if name == self.old_name and alias is not None + ] + if not aliases or info.get_imported_resource(context) != origin: + continue + stmt.import_info = importutils.FromImport( + info.module_name, + info.level, + [pair for pair in info.names_and_aliases if pair not in aliases], + ) + imports.add_import( + importutils.FromImport(self._new_modname(dest), 0, aliases) + ) + return imports.get_changed_source() + def _source_module_changes(self, dest): - placeholder = "__rope_moving_%s_" % self.old_name + placeholder = f"__rope_moving_{self.old_name}_" handle = _ChangeMoveOccurrencesHandle(placeholder) occurrence_finder = occurrences.create_finder( self.project, self.old_name, self.old_pyname @@ -412,7 +2626,7 @@ def _dest_module_changes(self, dest): pymodule = self.tools.new_pymodule(pymodule, source) moving, imports = self._get_moving_element_with_imports() - pymodule, has_changed = self._add_imports2(pymodule, imports) + pymodule, _has_changed = self._add_imports2(pymodule, imports) module_with_imports = self.import_tools.module_imports(pymodule) source = pymodule.source_code @@ -441,7 +2655,7 @@ def _dest_module_changes(self, dest): sort=False, selfs=False, unused=True, - import_filter=self._import_filter, + import_filter=lambda stmt: self._import_filter(stmt, dest.parent), ) return ChangeContents(dest, source) @@ -521,7 +2735,7 @@ def __init__(self, project, resource): def get_changes( self, dest: resources.Resource, - resources: Optional[List[resources.File]] = None, + resources: list[resources.File] | None = None, task_handle=taskhandle.DEFAULT_TASK_HANDLE, ): if resources is None: @@ -533,7 +2747,7 @@ def get_changes( return self._calculate_changes(dest, resources, task_handle) def _calculate_changes(self, dest, resources, task_handle): - changes = ChangeSet("Moving module <%s>" % self.old_name) + changes = ChangeSet(f"Moving module <{self.old_name}>") job_set = task_handle.create_jobset("Collecting changes", len(resources)) for module in resources: job_set.started_job(module.path) @@ -794,10 +3008,18 @@ def moving_code_with_imports(project, resource, source): origin = project.get_pymodule(resource) - imports = [stmt.import_info for stmt in import_tools.module_imports(origin).imports] - - back_names = [name for name in origin if name not in pymodule] - imports.append(import_tools.get_from_import(resource, back_names)) + origin_imports = import_tools.module_imports(origin) + used_names = origin_imports.get_used_names(pymodule) + imports = origin_imports.get_used_imports(pymodule) + back_names = [ + name + for name, binding in origin.get_attributes().items() + if name in used_names + and name not in pymodule + and not isinstance(binding, (pynames.ImportedName, pynames.ImportedModule)) + ] + if back_names: + imports.append(import_tools.get_from_import(resource, back_names)) source = _add_imports_to_module(import_tools, pymodule, imports) pymodule = libutils.get_string_module(project, source, resource) @@ -809,7 +3031,7 @@ def moving_code_with_imports(project, resource, source): # extracting imports after changes module_imports = import_tools.module_imports(pymodule) - imports = [import_stmt.import_info for import_stmt in module_imports.imports] + imports = module_imports.get_used_imports(pymodule) start = 1 if module_imports.imports: start = module_imports.imports[-1].end_line @@ -870,7 +3092,7 @@ def get_changed_module(self): self.skip_start, self.skip_end, self.replacement ) for occurrence in self.occurrence_finder.find_occurrences(self.resource): - start, end = occurrence.get_primary_range() + start, _end = occurrence.get_primary_range() if self.skip_start <= start < self.skip_end: self.handle.occurred_inside_skip(change_collector, occurrence) else: diff --git a/rope/refactor/occurrences.py b/rope/refactor/occurrences.py index 6bc3555bf..67e6b5bdf 100644 --- a/rope/refactor/occurrences.py +++ b/rope/refactor/occurrences.py @@ -40,7 +40,6 @@ from typing import Iterator from rope.base import ( - ast, codeanalyze, evaluate, exceptions, @@ -80,8 +79,21 @@ def find_occurrences(self, resource=None, pymodule=None): tools = _OccurrenceToolsCreator( self.project, resource=resource, pymodule=pymodule, docs=self.docs ) - for offset in self._textual_finder.find_offsets(tools.source_code): - occurrence = Occurrence(tools, offset) + quoted = ( + { + reference[0]: reference + for reference in tools.pymodule.get_source_binding_index().quoted_annotation_references( + self.name + ) + } + if self.name in tools.source_code or "\\" in tools.source_code + else {} + ) + offsets = ( + set(self._textual_finder.find_offsets(tools.source_code)) | quoted.keys() + ) + for offset in sorted(offsets): + occurrence = Occurrence(tools, offset, quoted.get(offset)) for filter in self.filters: result = filter(occurrence) if result is None: @@ -118,6 +130,10 @@ def create_finder( filters.append(NoImportsFilter()) if not keywords: filters.append(NoKeywordsFilter()) + from rope.base.sourcebindings import SourceValueName + + while isinstance(instance, SourceValueName): + instance = instance.source_pyname if isinstance(instance, pynames.ParameterName): for pyobject in instance.get_objects(): try: @@ -134,53 +150,80 @@ def create_finder( class Occurrence: - def __init__(self, tools, offset): + def __init__(self, tools, offset, quoted=None): self.tools = tools self.offset = offset self.resource = tools.resource + self.quoted = quoted @utils.saveit def get_word_range(self): + if self.quoted is not None: + return self.quoted[:2] return self.tools.word_finder.get_word_range(self.offset) @utils.saveit def get_primary_range(self): + if self.quoted is not None: + return self.quoted[2:4] return self.tools.word_finder.get_primary_range(self.offset) + def get_primary_text(self): + if self.quoted is not None: + return self.quoted[5] + start, end = self.get_primary_range() + return self.tools.source_code[start:end] + @utils.saveit def get_pyname(self): + if self.quoted is not None: + return self.get_primary_and_pyname()[1] with contextlib.suppress(exceptions.BadIdentifierError): return self.tools.name_finder.get_pyname_at(self.offset) @utils.saveit def get_primary_and_pyname(self): + if self.quoted is not None: + return evaluate.eval_str2(self.quoted[4].environment, self.quoted[5]) with contextlib.suppress(exceptions.BadIdentifierError): return self.tools.name_finder.get_primary_and_pyname_at(self.offset) @utils.saveit def is_in_import_statement(self): - return self.tools.word_finder.is_from_statement( - self.offset - ) or self.tools.word_finder.is_import_statement(self.offset) + return any( + start <= self.offset < end + for layout in self.tools.pymodule.get_source_binding_index().import_layouts() + for start, end in (layout.region,) + ) def is_called(self): + if self.quoted is not None: + return False return self.tools.word_finder.is_a_function_being_called(self.offset) def is_defined(self): + if self.quoted is not None: + return False return self.tools.word_finder.is_a_class_or_function_name_in_header(self.offset) def is_a_fixed_primary(self): + if self.quoted is not None: + return False return self.tools.word_finder.is_a_class_or_function_name_in_header( self.offset ) or self.tools.word_finder.is_a_name_after_from_import(self.offset) def is_written(self): + if self.quoted is not None: + return False return self.tools.word_finder.is_assigned_here(self.offset) def is_unsure(self): return unsure_pyname(self.get_pyname()) def is_function_keyword_parameter(self): + if self.quoted is not None: + return False return self.tools.word_finder.is_function_keyword_parameter(self.offset) @property @@ -194,6 +237,8 @@ def same_pyname(expected, pyname): """Check whether `expected` and `pyname` are the same""" if expected is None or pyname is None: return False + expected = getattr(expected, "source_pyname", expected) + pyname = getattr(pyname, "source_pyname", pyname) if expected == pyname: return True if not isinstance( @@ -214,12 +259,42 @@ def unsure_pyname(pyname, unbound=True): """Return `True` if we don't know what this name references""" if pyname is None: return True + if not getattr(pyname, "is_refactoring_identity_safe", True): + return True if unbound and not isinstance(pyname, pynames.UnboundName): return False if pyname.get_object() == pyobjects.get_unknown(): return True +def check_opaque_references(project, pyname, resources, region=None, name=None): + if not getattr(pyname, "is_refactoring_identity_safe", True): + raise exceptions.RefactoringError( + "Cannot refactor a name with an unresolved source binding." + ) + for resource in resources: + # Opaque references are Name nodes in annotation contexts; a module + # whose source never spells the name cannot hold one, and building + # its full source-binding index just to prove that is wasted work. + if name is not None: + source = resource.read() + if not re.search(rf"\b{re.escape(name)}\b", source): + continue + module = project.get_pymodule(resource) + index = module.get_source_binding_index() + for reference in index.opaque_references(): + if region is not None: + start, _ = codeanalyze.ASTLinesAdapter(module.source_code)[ + reference.node + ] + if not region[0] <= start < region[1]: + continue + if reference.may_reference(pyname): + raise exceptions.RefactoringError( + "Cannot refactor a name possibly referenced in an unresolved annotation binding." + ) + + class PyNameFilter: """For finding occurrences of a name.""" @@ -311,14 +386,6 @@ class _TextualFinder: def __init__(self, name, docs=False): self.name = name self.docs = docs - self.comment_pattern = _TextualFinder.any("comment", [r"#[^\n]*"]) - self.string_pattern = _TextualFinder.any( - "string", [codeanalyze.get_string_pattern()] - ) - self.f_string_pattern = _TextualFinder.any( - "fstring", [codeanalyze.get_formatted_string_pattern()] - ) - self.pattern = self._get_occurrence_pattern(self.name) def find_offsets(self, source: str) -> Iterator[int]: if not self._fast_file_query(source): @@ -326,26 +393,12 @@ def find_offsets(self, source: str) -> Iterator[int]: if self.docs: searcher = self._normal_search else: - searcher = self._re_search + searcher = self._token_search yield from searcher(source) - def _re_search(self, source: str) -> Iterator[int]: - for match in self.pattern.finditer(source): - if match.groupdict()["occurrence"]: - yield match.start("occurrence") - elif match.groupdict()["fstring"]: - f_string = match.groupdict()["fstring"] - for offset in self._search_in_f_string(f_string): - yield match.start("fstring") + offset - - def _search_in_f_string(self, f_string: str) -> Iterator[int]: - tree = ast.parse(f_string) - for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id == self.name: - yield node.col_offset - elif isinstance(node, ast.Attribute) and node.attr == self.name: - assert node.end_col_offset is not None - yield node.end_col_offset - len(self.name) + def _token_search(self, source: str) -> Iterator[int]: + """Yield the offsets of every NAME token equal to the searched name.""" + yield from codeanalyze.name_offsets(source, self.name) def _normal_search(self, source: str) -> Iterator[int]: current = 0 @@ -372,27 +425,6 @@ def _get_source(self, resource, pymodule): else: return pymodule.source_code - def _get_occurrence_pattern(self, name): - # A name directly followed by a quote is a string prefix, e.g. the - # ``f`` in ``f"..."``, never a reference to the name. - occurrence_pattern = _TextualFinder.any( - "occurrence", ["\\b" + name + "\\b(?![\"'])"] - ) - pattern = re.compile( - occurrence_pattern - + "|" - + self.comment_pattern - + "|" - + self.string_pattern - + "|" - + self.f_string_pattern - ) - return pattern - - @staticmethod - def any(name, list_): - return "(?P<%s>" % name + "|".join(list_) + ")" - class _OccurrenceToolsCreator: def __init__(self, project, resource=None, pymodule=None, docs=False): diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index fbebceab2..6067a9988 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -1,6 +1,9 @@ +import bisect import collections +import io import numbers import re +import tokenize import warnings from itertools import chain @@ -317,9 +320,13 @@ def _ClassDef(self, node): children.extend(("@", decorator)) children.extend(["class", node.name]) self._add_type_params(node, children) - if node.bases: + if node.bases or node.keywords: children.append("(") - children.extend(self._child_nodes(node.bases, ",")) + arguments = sorted( + [*node.bases, *node.keywords], + key=lambda argument: (argument.lineno, argument.col_offset), + ) + children.extend(self._child_nodes(arguments, ",")) children.append(")") children.append(":") children.extend(node.body) @@ -350,14 +357,16 @@ def start_quote_char(): return self.source[start : quote_pos + len(quote_char)] def end_quote_char(): - possible_quotes = [ - (self.source.source.rfind(q, start, end), q) - for q in reversed(QUOTE_CHARS) - ] - _, quote_pos, quote_char = max( - (len(q), pos, q) for pos, q in possible_quotes if pos != -1 + # The delimiter is whatever quote sequence terminates the + # literal, not the longest quote found anywhere in the body + # (a double-quoted f-string may contain a ''' sequence). + for quote_char in QUOTE_CHARS: + if self.source.source[end - len(quote_char) : end] == quote_char: + return quote_char + raise MismatchedTokenError( + f"f-string at line {self.lines.get_line_number(start)} does not" + " end with a quote delimiter" ) - return self.source[end - len(quote_char) : end] QUOTE_CHARS = ['"""', "'''", '"', "'"] offset = self.source.offset @@ -370,8 +379,39 @@ def end_quote_char(): children.append(part) children.append(end_quote_char()) self._handle(node, children) + self._extend_joined_string_to(node, end) + + def _extend_joined_string_to(self, node, string_end): + """Cover concatenation parts that follow the last interpolation. + + `_handle` stops at the first closing quote it finds after the last + `FormattedValue`, so in an implicit concatenation whose trailing parts + are pure literal text -- ``f"{a} one " f"#52 two"`` -- it stops at the + end of the part holding the interpolation. That both understates the + node's region and parks the scanner inside the literal, where an + unescaped ``#`` reads as the start of a comment and swallows the rest + of the line. `consume_string` already measured the whole + concatenation, so close the gap from its end offset. + """ + if self.source.offset >= string_end: + return + trailing_parts = self.source[self.source.offset : string_end] + if self.children: + node.sorted_children.append(trailing_parts) + self.source.offset = string_end + node.region = (node.region[0], string_end) def _FormattedValue(self, node): + # Expression positions distinguish real fields from escaped literal braces. + value_start, _ = self.ast_adapter[node.value] + if value_start is None: + raise MismatchedTokenError("F-string expression AST has no source position") + field_start = self.source.rfind_token("{", self.source.offset, value_start) + if field_start is None: + raise MismatchedTokenError( + "F-string expression has no matching opening brace" + ) + self.source.offset = field_start children = [] children.append("{") children.append(node.value) @@ -454,6 +494,8 @@ def _handle_function_def_node(self, node, is_async): children.append(node.name) self._add_type_params(node, children) children.extend(["(", node.args, ")"]) + if node.returns is not None: + children.extend(["->", node.returns]) children.append(":") children.extend(node.body) self._handle(node, children) @@ -466,22 +508,34 @@ def _AsyncFunctionDef(self, node): def _arguments(self, node): children = [] - args = list(node.args) - defaults = [None] * (len(args) - len(node.defaults)) + list(node.defaults) - for index, (arg, default) in enumerate(zip(args, defaults)): + positional = list(node.posonlyargs) + list(node.args) + defaults = [None] * (len(positional) - len(node.defaults)) + list(node.defaults) + for index, (arg, default) in enumerate(zip(positional, defaults)): if index > 0: children.append(",") self._add_args_to_children(children, arg, default) - if node.vararg is not None: - if args: + if node.posonlyargs and index + 1 == len(node.posonlyargs): + children.extend([",", "/"]) + if node.vararg is not None or node.kwonlyargs: + if positional: children.append(",") - children.extend(["*", node.vararg.arg]) + children.append("*") + if node.vararg is not None: + children.append(self._star_parameter(node.vararg)) + for arg, default in zip(node.kwonlyargs, node.kw_defaults): + children.append(",") + self._add_args_to_children(children, arg, default) if node.kwarg is not None: - if args or node.vararg is not None: + if positional or node.vararg is not None or node.kwonlyargs: children.append(",") - children.extend(["**", node.kwarg.arg]) + children.extend(["**", self._star_parameter(node.kwarg)]) self._handle(node, children) + @staticmethod + def _star_parameter(arg): + """An annotated ``*args``/``**kwargs`` is walked as a node, else by name.""" + return arg if arg.annotation is not None else arg.arg + def _add_args_to_children(self, children, arg, default): if isinstance(arg, (list, tuple)): self._add_tuple_parameter(children, arg) @@ -590,7 +644,10 @@ def _NameConstant(self, node): self._handle(node, [str(node.value)]) def _arg(self, node): - self._handle(node, [node.arg]) + children = [node.arg] + if node.annotation is not None: + children.extend([":", node.annotation]) + self._handle(node, children) def _Pass(self, node): self._handle(node, ["pass"]) @@ -751,24 +808,32 @@ def _MatchSequence(self, node): self._handle(node, children) def _get_surrounding_parens(self, node: ast.MatchSequence): + """The sequence's own ``[...]``/``(...)`` delimiters, if it has them. + + The text between the node's start and its first pattern can also + hold grouping parentheses of that pattern -- ``case (("a" | b), c)`` + -- which the AST does not represent. The sequence is delimited only + when the bracket at its first character is the one closed at its + last character; ``case (1), (2)`` starts and ends with parentheses + that belong to its elements. + """ node_start, node_end = self.ast_adapter[node] - first_pattern_start, _ = self.ast_adapter[node.patterns[0]] - _, last_pattern_end = self.ast_adapter[node.patterns[-1]] - opening_paren = self.source[node_start:first_pattern_start].strip() - closing_paren = self.source[last_pattern_end:node_end].strip() - - if opening_paren not in ["[", "(", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!", - RuntimeWarning, - ) - - if closing_paren not in ["]", ")", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!", - RuntimeWarning, - ) - return opening_paren, closing_paren + segment = self.source[node_start:node_end] + pairs = {"[": "]", "(": ")"} + if segment[:1] not in pairs or segment[-1:] != pairs[segment[0]]: + return "", "" + depth = 0 + for token in tokenize.generate_tokens(io.StringIO(segment).readline): + if token.type != tokenize.OP or token.string not in "()[]{}": + continue + depth += 1 if token.string in "([{" else -1 + if depth == 0: + end_row = segment.count("\n") + 1 + end_col = len(segment) - (segment.rfind("\n") + 1) + if token.end != (end_row, end_col): + return "", "" + return segment[0], segment[-1] + return "", "" def _MatchStar(self, node): self._handle(node, ["*", node.name or "_"]) @@ -845,15 +910,21 @@ def __init__(self, source): self.source = source self.offset = 0 self.ast_adapter = codeanalyze.ASTLinesAdapter(source) + self._ranges = None def consume(self, token, skip_comment=True): try: while True: new_offset = self.source.index(token, self.offset) - if self._good_token(token, new_offset) or not skip_comment: + if not skip_comment: break - else: - self._skip_comment() + here = self._enclosing_literal(self.offset) + there = self._enclosing_literal(new_offset) + if there is None or there == here: + break + # The match is buried in a literal the walker is not + # itself working inside. Resume past that literal. + self.offset = there[1] except (ValueError, TypeError): raise MismatchedTokenError( f"Token <{token}> at {self._get_location()} cannot be matched" @@ -868,23 +939,86 @@ def consume_joined_string(self, token): def consume_node(self, node): start, end = self.ast_adapter[node] - if self.offset < end: - self.offset = end + self.offset = max(self.offset, end) return start, end - def _good_token(self, token, offset, start=None): + def _good_token(self, token, offset): """Checks whether consumed token is in comments""" - if start is None: - start = self.offset - try: - comment_index = self.source.rindex("#", start, offset) - except ValueError: - return True + literal = self._enclosing_literal(offset) + return literal is None or not literal[2] + + def _enclosing_literal(self, offset): + """The string literal or comment containing ``offset``, if any. + + A ``(start, end, is_comment)`` triple, or ``None`` when + ``offset`` sits in ordinary code. + + The walker searches the source as plain text, which cannot tell + a token from the same characters inside a literal. Asked for + ``)`` it would match the one in ``f"see (#123)"``, or the one in + a triple-quoted ``CREATE TABLE t (id)`` blob, then carry on + from a position the tree knows nothing about -- surfacing much + later as an error naming an unrelated node. + """ + ranges, starts = self._get_ranges() + index = bisect.bisect_right(starts, offset) - 1 + if index >= 0 and ranges[index][1] > offset: + return ranges[index] + return None + + def _get_ranges(self): + """Every string literal and comment, as sorted offset ranges. + + Paired with the list of their start offsets, to bisect on. + + Tokenized once per source. The tokenizer is what makes this + exact rather than another guess at where a literal starts: a + scan for quotes cannot know whether it began inside one. + + On an unparseable source it yields nothing, leaving the plain + textual search of before. + """ + if self._ranges is None: + ranges = self._tokenize_ranges() + self._ranges = (ranges, [start for start, _, _ in ranges]) + return self._ranges + + def _tokenize_ranges(self): + line_starts = [0] + for line in self.source.splitlines(keepends=True): + line_starts.append(line_starts[-1] + len(line)) + + def offset_of(position): + row, column = position + if row >= len(line_starts): + return len(self.source) + return line_starts[row - 1] + column + + # FSTRING_* exists from Python 3.12 on, where an f-string is + # tokenized in pieces. Taking only MIDDLE keeps the replacement + # fields out: they are code, and the walker consumes them. + # Before 3.12 an f-string is one STRING token, fields included -- + # which is why `consume` compares the walker's own literal to the + # match's rather than skipping every literal it meets. + wanted = {tokenize.STRING, tokenize.COMMENT} + for name in ("FSTRING_START", "FSTRING_MIDDLE", "FSTRING_END"): + if hasattr(tokenize, name): + wanted.add(getattr(tokenize, name)) + ranges = [] try: - new_line_index = self.source.rindex("\n", start, offset) - except ValueError: - return False - return comment_index < new_line_index + tokens = tokenize.generate_tokens(io.StringIO(self.source).readline) + for token in tokens: + if token.type in wanted: + ranges.append( + ( + offset_of(token.start), + offset_of(token.end), + token.type == tokenize.COMMENT, + ) + ) + except (tokenize.TokenError, IndentationError, SyntaxError, ValueError): + return [] + return ranges def _skip_comment(self): self.offset = self.source.index("\n", self.offset + 1) @@ -898,7 +1032,7 @@ def rfind_token(self, token, start, end): while True: try: index = self.source.rindex(token, start, end) - if self._good_token(token, index, start=start): + if self._good_token(token, index): return index else: end = index diff --git a/rope/refactor/rename.py b/rope/refactor/rename.py index fcdd17964..57ef610ff 100644 --- a/rope/refactor/rename.py +++ b/rope/refactor/rename.py @@ -2,6 +2,7 @@ from keyword import iskeyword from rope.base import ( + ast, codeanalyze, evaluate, exceptions, @@ -104,9 +105,28 @@ def unsure_func(value=unsure): resources = [self.resource] if _is_local(self.old_pyname): resources = [self.resource] + all_project_resources = resources is None if resources is None: resources = self.project.get_python_files() + resources = list(resources) self.validate_changes(new_name) + occurrences.check_opaque_references( + self.project, self.old_pyname, resources, name=self.old_name + ) + from rope.base import externaldata + + defining, _ = self.old_pyname.get_definition_location() + if defining is not None: + externaldata.check_rename( + defining, self.old_name, self.old_pyname, self.old_instance + ) + for file_ in resources: + externaldata.check_rename( + self.project.get_pymodule(file_), + self.old_name, + self.old_pyname, + self.old_instance, + ) changes = ChangeSet(f"Renaming <{self.old_name}> to <{new_name}>") finder = occurrences.create_finder( self.project, @@ -120,16 +140,48 @@ def unsure_func(value=unsure): job_set = task_handle.create_jobset("Collecting Changes", len(resources)) for file_ in resources: job_set.started_job(file_.path) - new_content = rename_in_module(finder, new_name, resource=file_) + new_content = rename_in_module( + finder, + new_name, + resource=file_, + preserve_imports=self._get_preserved_imports(file_), + ) if new_content is not None: changes.add_change(ChangeContents(file_, new_content)) job_set.finished_job() if self._is_renaming_a_module(): resource = self.old_pyname.get_object().get_resource() - if self._is_allowed_to_move(resources, resource): + if self._is_allowed_to_move(resources, resource, all_project_resources): self._rename_module(resource, new_name, changes) return changes + def _get_preserved_imports(self, resource): + """Find imported names whose definitions are outside the project.""" + pymodule = self.project.get_pymodule(resource) + lines = codeanalyze.ASTLinesAdapter(pymodule.source_code) + for node in ast.walk(pymodule.get_ast()): + if not isinstance(node, ast.ImportFrom): + continue + names = [alias for alias in node.names if alias.name == self.old_name] + if not names: + continue + imported_module = pynames.ImportedModule( + pymodule, node.module or "", level=node.level + ).get_object() + imported_resource = ( + imported_module.get_resource() + if isinstance(imported_module, pyobjects.AbstractModule) + else None + ) + if imported_resource is not None and ( + imported_resource.project == self.project + and not self.project.is_ignored(imported_resource) + ): + continue + for alias in names: + start, _ = lines[alias] + yield start, alias.asname is None + def validate_changes( self, new_name: str, @@ -140,12 +192,21 @@ def validate_changes( f"Invalid refactoring target name. '{new_name}' is a Python keyword." ) - def _is_allowed_to_move(self, resources, resource): + def _is_allowed_to_move(self, resources, resource, all_project_resources=False): if resource.is_folder(): try: return resource.get_child("__init__.py") in resources except exceptions.ResourceNotFoundError: - return False + if all_project_resources: + return True + package_files = [ + file + for file in self.project.get_python_files() + if resource.contains(file) + ] + return bool(package_files) and all( + file in resources for file in package_files + ) else: return resource in resources @@ -197,12 +258,27 @@ def get_old_name(self): return word_finder.get_primary_at(self.offset) def _get_scope_offset(self): - scope = self.pymodule.get_scope().get_inner_scope_for_offset(self.offset) + scope = getattr(self.old_pyname, "binding_scope", None) + if scope is None or scope.get_kind() not in { + "Lambda", + "Comprehension", + "TypeParameters", + }: + scope = self.pymodule.get_scope().get_structural_scope_for_offset( + self.offset + ) return scope.get_region() def get_changes(self, new_name, only_calls=False, reads=True, writes=True): changes = ChangeSet(f"Changing <{self.old_name}> occurrences to <{new_name}>") scope_start, scope_end = self._get_scope_offset() + occurrences.check_opaque_references( + self.project, + self.old_pyname, + [self.resource], + (scope_start, scope_end), + name=self.old_name, + ) finder = occurrences.create_finder( self.project, self.old_name, @@ -233,6 +309,7 @@ def rename_in_module( region=None, reads=True, writes=True, + preserve_imports=None, ): """Returns the changed source or `None` if there is no changes""" if resource is not None: @@ -240,6 +317,7 @@ def rename_in_module( else: source_code = pymodule.source_code change_collector = codeanalyze.ChangeCollector(source_code) + preserved_imports = None for occurrence in occurrences_finder.find_occurrences(resource, pymodule): if replace_primary and occurrence.is_a_fixed_primary(): continue @@ -252,22 +330,34 @@ def rename_in_module( ): continue if region is None or region[0] <= start < region[1]: - change_collector.add_change(start, end, new_name) + replacement = new_name + if preserve_imports is not None and occurrence.is_in_import_statement(): + if preserved_imports is None: + preserved_imports = dict(preserve_imports) + if start in preserved_imports: + if not preserved_imports[start]: + continue + if source_code[start:end] != new_name: + replacement = source_code[start:end] + " as " + new_name + change_collector.add_change(start, end, replacement) return change_collector.get_changed() def _is_local(pyname): + binding_scope = getattr(pyname, "binding_scope", None) + if binding_scope is not None: + return not isinstance( + pyname, pynames.ParameterName + ) and binding_scope.get_kind() in ( + "Function", + "Lambda", + "Comprehension", + "TypeParameters", + ) + if not isinstance(pyname, pynames.AssignedName): + return False module, lineno = pyname.get_definition_location() if lineno is None: return False scope = module.get_scope().get_inner_scope_for_line(lineno) - if isinstance(pyname, pynames.DefinedName) and scope.get_kind() in ( - "Function", - "Class", - ): - scope = scope.parent - return ( - scope.get_kind() == "Function" - and pyname in scope.get_names().values() - and isinstance(pyname, pynames.AssignedName) - ) + return scope.get_kind() == "Function" and pyname in scope.get_names().values() diff --git a/rope/refactor/restructure.py b/rope/refactor/restructure.py index 55a4b2d03..9af935eca 100644 --- a/rope/refactor/restructure.py +++ b/rope/refactor/restructure.py @@ -1,9 +1,17 @@ +from __future__ import annotations + import warnings +from typing import Callable, List, Optional, Sequence from rope.base import ast, builtins, change, codeanalyze, libutils, taskhandle +from rope.base.pyobjectsdef import PyModule +from rope.base.resources import File from rope.refactor import patchedast, similarfinder, sourceutils from rope.refactor.importutils import module_imports +MatchFilter = Callable[[PyModule, similarfinder.Match], bool] +ImportFilter = Callable[[File, List[str]], Sequence[str]] + class Restructure: """A class to perform python restructurings @@ -71,11 +79,29 @@ class Restructure: """ - def __init__(self, project, pattern, goal, args=None, imports=None, wildcards=None): + def __init__( + self, + project, + pattern, + goal, + args=None, + imports=None, + wildcards=None, + match_filter: Optional[MatchFilter] = None, + import_filter: Optional[ImportFilter] = None, + ): """Construct a restructuring See class pydoc for more info about the arguments. + `match_filter`, when given, is called as ``match_filter(pymodule, + match)`` for every match found in a module; only the matches it + returns a true value for are restructured. + + `import_filter`, when given, is called as ``import_filter(resource, + imports)`` before `imports` are added to a changed module; only the + imports it returns are added. + """ self.project = project self.pattern = pattern @@ -87,6 +113,8 @@ def __init__(self, project, pattern, goal, args=None, imports=None, wildcards=No if self.imports is None: self.imports = [] self.wildcards = wildcards + self.match_filter = match_filter + self.import_filter = import_filter self.template = similarfinder.CodeTemplate(self.goal) def get_changes( @@ -150,6 +178,10 @@ def get_changes( pymodule = self.project.get_pymodule(resource) finder = similarfinder.SimilarFinder(pymodule, wildcards=self.wildcards) matches = list(finder.get_matches(self.pattern, self.args)) + if self.match_filter is not None: + matches = [ + match for match in matches if self.match_filter(pymodule, match) + ] computer = self._compute_changes(matches, pymodule) result = computer.get_changed() if result is not None: @@ -168,6 +200,8 @@ def _compute_changes(self, matches, pymodule): ) def _add_imports(self, resource, source, imports): + if self.import_filter is not None: + imports = list(self.import_filter(resource, list(imports))) if not imports: return source import_infos = self._get_import_infos(resource, imports) diff --git a/rope/refactor/similarfinder.py b/rope/refactor/similarfinder.py index dd5bec362..4309ce682 100644 --- a/rope/refactor/similarfinder.py +++ b/rope/refactor/similarfinder.py @@ -154,8 +154,8 @@ def __init__(self, body, pattern, does_match): def find_matches(self): if self.matches is None: self.matches = [] - # _check_nodes always returns None, so - # call_for_nodes traverses self.body's entire tree. + # call_for_nodes traverses self.body's entire tree, except the + # inside of f-strings, which _check_node walks itself. ast.call_for_nodes(self.body, self._check_node) return self.matches @@ -164,6 +164,20 @@ def _check_node(self, node): self._check_statements(node) else: self._check_expression(node) + if isinstance(node, ast.JoinedStr): + self._check_replacement_fields(node) + return True + return False + + def _check_replacement_fields(self, node): + # The literal pieces and format specs of an f-string are fragments of + # its source, not expressions: patchedast gives them no region. Only + # the expressions in the replacement fields can match. + for value in node.values: + if isinstance(value, ast.FormattedValue): + ast.call_for_nodes(value.value, self._check_node) + if value.format_spec is not None: + self._check_replacement_fields(value.format_spec) def _check_expression(self, node): mapping = {} @@ -249,6 +263,12 @@ def get_ast(self, name): """Return the ast node that has matched rope variables""" return self.mapping.get(name, None) + def get_capture_region(self, name): + """Return the captured variable's half-open source character range.""" + if name not in self.mapping: + raise ValueError("Match has no captured variable: %s" % name) + return patchedast.node_region(self.mapping[name]) + class ExpressionMatch(Match): def __init__(self, ast, mapping): @@ -269,10 +289,26 @@ def get_region(self): class CodeTemplate: + _dollar_name_pattern = r"(?P\$\{[^\s\$\}]*\})" + def __init__(self, template): self.template = template self._find_names() + @classmethod + def _get_pattern(cls): + if cls._match_pattern is None: + pattern = "|".join( + ( + codeanalyze.get_comment_pattern(), + codeanalyze.get_string_pattern(), + f"(?P{codeanalyze.get_formatted_string_pattern()})", + cls._dollar_name_pattern, + ) + ) + cls._match_pattern = re.compile(pattern) + return cls._match_pattern + def _find_names(self): self.names = {} for match in CodeTemplate._get_pattern().finditer(self.template): @@ -283,6 +319,29 @@ def _find_names(self): self.names[name] = [] self.names[name].append((start, end)) + elif "fstring" in match.groupdict() and match.group("fstring") is not None: + self._fstring_case(match) + + def _fstring_case(self, fstring_match: re.Match): + """Needed because CodeTemplate._match_pattern short circuits + as soon as it sees a '#', even if that '#' is inside a f-string + that has a ${variable}.""" + + string_start, string_end = fstring_match.span("fstring") + + for match in re.finditer(self._dollar_name_pattern, self.template): + if match.start("name") < string_start: + continue + + if match.end("name") > string_end: + break + + start, end = match.span("name") + name = self.template[start + 2 : end - 1] + if name not in self.names: + self.names[name] = [] + self.names[name].append((start, end)) + def get_names(self): return self.names.keys() @@ -298,19 +357,6 @@ def substitute(self, mapping): _match_pattern = None - @classmethod - def _get_pattern(cls): - if cls._match_pattern is None: - pattern = ( - codeanalyze.get_comment_pattern() - + "|" - + codeanalyze.get_string_pattern() - + "|" - + r"(?P\$\{[^\s\$\}]*\})" - ) - cls._match_pattern = re.compile(pattern) - return cls._match_pattern - class _RopeVariable: """Transform and identify rope inserted wildcards""" diff --git a/rope/refactor/sourceutils.py b/rope/refactor/sourceutils.py index d55aad4c9..3bfb021ed 100644 --- a/rope/refactor/sourceutils.py +++ b/rope/refactor/sourceutils.py @@ -38,11 +38,11 @@ def fix_indentation(code: str, new_indents: int) -> str: return indent_lines(code, new_indents - min_indents) -def add_methods(pymodule, class_scope, methods_sources): +def add_methods(pymodule, class_scope, methods_sources, *, at_end=False): source_code = pymodule.source_code lines = pymodule.lines insertion_line = class_scope.get_end() - if class_scope.get_scopes(): + if not at_end and class_scope.get_scopes(): insertion_line = class_scope.get_scopes()[-1].get_end() insertion_offset = lines.get_line_end(insertion_line) methods = "\n\n" + "\n\n".join(methods_sources) diff --git a/rope/refactor/usefunction.py b/rope/refactor/usefunction.py index cf1330e80..fdc60562b 100644 --- a/rope/refactor/usefunction.py +++ b/rope/refactor/usefunction.py @@ -48,6 +48,10 @@ def _check_returns(self): ) def get_changes(self, resources=None, task_handle=taskhandle.DEFAULT_TASK_HANDLE): + if self.pyfunction.get_ast().args.kwonlyargs: + raise exceptions.RefactoringError( + "Use function does not support keyword-only parameters." + ) if resources is None: resources = self.project.get_python_files() changes = change.ChangeSet("Using function <%s>" % self.pyfunction.get_name()) @@ -192,6 +196,7 @@ def _Return(self, node): def _NamedExpr(self, node): self.named_expression += 1 + self.visit(node.value) def _Yield(self, node): self.yields += 1 diff --git a/ropetest-package-fixtures/external_fixturepkg/src/external_fixturepkg/__main__.py b/ropetest-package-fixtures/external_fixturepkg/src/external_fixturepkg/__main__.py index e6d098dee..4693850cb 100644 --- a/ropetest-package-fixtures/external_fixturepkg/src/external_fixturepkg/__main__.py +++ b/ropetest-package-fixtures/external_fixturepkg/src/external_fixturepkg/__main__.py @@ -2,5 +2,5 @@ def main() -> None: pass -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/ropetest/advanced_oi_test.py b/ropetest/advanced_oi_test.py index 79f514ad0..c78aadd08 100644 --- a/ropetest/advanced_oi_test.py +++ b/ropetest/advanced_oi_test.py @@ -329,6 +329,37 @@ def complex_to_textual(pyobject): self.assertEqual(to_textual.transform(var), result) self.assertEqual(complex_to_textual(var), result) + def test_builtin_instance_textual_transformations(self): + mod = testutils.create_module(self.project, "mod") + mod.write(dedent("""\ + an_int = int() + an_object = object() + ints = [an_int] + """)) + to_pyobject = rope.base.oi.transform.TextualToPyObject(self.project) + to_textual = rope.base.oi.transform.PyObjectToTextual(self.project) + pymod = self.project.get_pymodule(mod) + for name, builtin in ( + ("an_int", "int"), + ("an_object", "object"), + ): + pyobject = pymod[name].get_object() + textual = to_textual.transform(pyobject) + self.assertEqual(("builtin", "instance", builtin), textual) + restored = to_pyobject.transform(textual) + self.assertIs(pyobject.get_type(), restored.get_type()) + ints = pymod["ints"].get_object() + self.assertEqual( + ("builtin", "list", ("builtin", "instance", "int")), + to_textual.transform(ints), + ) + read = rope.base.pyobjects.sampled_value(pymod["an_int"].get_object(), ints) + self.assertEqual(("unknown",), to_textual.transform(read)) + self.assertEqual( + ("builtin", "list", ("builtin", "instance", "int")), + to_textual.transform(rope.base.builtins.get_list(read)), + ) + def test_arguments_with_keywords(self): mod = testutils.create_module(self.project, "mod") code = dedent("""\ @@ -507,6 +538,78 @@ def f(p): p_type = f_scope["p"].get_object().get_type() self.assertEqual(c_class, p_type) + def _parameter_type(self, pymod, path, parameter): + scope = pymod[path[0]].get_object() + for part in path[1:]: + scope = scope[part].get_object() + return scope.get_scope()[parameter].get_object().get_type() + + def test_static_oi_skips_reflected_operator_after_forward_result(self): + code = dedent("""\ + class Left(object): + def __add__(self, other): + return Left() + class Right(object): + def __radd__(self, other): + return other + Left() + Right() + """) + self.mod.write(code) + self.pycore.analyze_module(self.mod) + pymod = self.project.get_pymodule(self.mod) + self.assertEqual( + pymod["Right"].get_object(), + self._parameter_type(pymod, ("Left", "__add__"), "other"), + ) + self.assertIsNot( + pymod["Left"].get_object(), + self._parameter_type(pymod, ("Right", "__radd__"), "other"), + ) + + def test_static_oi_membership_compares_builtin_iterator_items(self): + code = dedent("""\ + class Item(object): + def __eq__(self, other): + return True + class Needle(object): + pass + class Items(object): + def __iter__(self): + return iter([Item()]) + Needle() in Items() + """) + self.mod.write(code) + self.pycore.analyze_module(self.mod) + pymod = self.project.get_pymodule(self.mod) + self.assertEqual( + pymod["Needle"].get_object(), + self._parameter_type(pymod, ("Item", "__eq__"), "other"), + ) + + def test_static_oi_class_attribute_store_uses_metaclass_setattr(self): + code = dedent("""\ + class Value(object): + pass + class Meta(type): + def __setattr__(cls, name, value): + type.__setattr__(cls, name, value) + class Owner(metaclass=Meta): + def __setattr__(self, name, value): + object.__setattr__(self, name, value) + Owner.attribute = Value() + """) + self.mod.write(code) + self.pycore.analyze_module(self.mod) + pymod = self.project.get_pymodule(self.mod) + self.assertEqual( + pymod["Value"].get_object(), + self._parameter_type(pymod, ("Meta", "__setattr__"), "value"), + ) + self.assertIsNot( + pymod["Value"].get_object(), + self._parameter_type(pymod, ("Owner", "__setattr__"), "value"), + ) + def test_static_oi_not_failing_when_callin_callables(self): code = dedent("""\ class C(object): diff --git a/ropetest/builtinstest.py b/ropetest/builtinstest.py index b21601803..e05892cbe 100644 --- a/ropetest/builtinstest.py +++ b/ropetest/builtinstest.py @@ -7,6 +7,147 @@ class BuiltinTypesTest(unittest.TestCase): + def test_generic_alias_inequality_preserves_not_implemented(self): + code = "Alias = list[str]\nSame = list[str]\n" + namespace = {} + exec(code, namespace) + self.assertEqual(namespace['Alias'], namespace['Same']) + self.assertNotEqual(namespace['Alias'], object()) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + alias = module['Alias'].get_object() + same = module['Same'].get_object() + self.assertIsInstance(alias, builtins.GenericAlias) + self.assertEqual(alias, same) + self.assertFalse(alias != same) + self.assertIs(NotImplemented, alias.__ne__(pyobjects.get_unknown())) + self.assertNotEqual(alias, pyobjects.get_unknown()) + + def test_native_getattribute_preserves_callable_descriptor_binding(self): + cases = [ + ("object.__getattribute__(Owner(), 'read')()", "Value"), + ("type.__getattribute__(Owner, 'read')(Owner())", "Value"), + ("object.__getattribute__(Owner(), 'static')()", "Value"), + ("type.__getattribute__(Owner, 'static')()", "Value"), + ("object.__getattribute__(Owner(), 'build')()", "Owner"), + ("type.__getattribute__(Owner, 'build')()", "Owner"), + ("getattr(Owner(), 'read')()", "Value"), + ("getattr(Owner, 'read')(Owner())", "Value"), + ("getattr(Owner(), 'static')()", "Value"), + ("getattr(Owner, 'build')()", "Owner"), + ] + for expression, expected in cases: + with self.subTest(expression=expression): + code = ( + "class Value:\n pass\n\n" + "class Owner:\n def read(self):\n return Value()\n" + " @staticmethod\n def static():\n return Value()\n" + " @classmethod\n def build(cls):\n return cls()\n\n" + + "result = " + expression + "\n" + ) + namespace = {} + exec(code, namespace) + self.assertIsInstance(namespace['result'], namespace[expected]) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIs(module[expected].get_object(), module['result'].get_object().get_type()) + + def test_native_getattribute_unknown_member_has_no_invented_result(self): + code = "class Owner:\n pass\n\nresult = object.__getattribute__(Owner(), 'missing')\n" + with self.assertRaises(AttributeError): + exec(code, {}) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIs(pyobjects.get_unknown(), module['result'].get_object()) + + def test_getattr_default_for_absent_member(self): + cases = [ + ("getattr(Owner(), 'missing', Value())", "Value"), + ("getattr(Owner, 'missing', Value())", "Value"), + ("getattr(Owner(), 'read', Other())()", "Value"), + ] + for expression, expected in cases: + with self.subTest(expression=expression): + code = ( + "class Value:\n pass\n\nclass Other:\n pass\n\n" + "class Owner:\n def read(self):\n return Value()\n\n" + + "result = " + expression + "\n" + ) + namespace = {} + exec(code, namespace) + self.assertIsInstance(namespace['result'], namespace[expected]) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIs(module[expected].get_object(), module['result'].get_object().get_type()) + + def test_getattr_dynamic_hook_has_no_invented_callable_identity(self): + code = ( + "class Owner:\n" + " def __getattribute__(self, name):\n return lambda: 7\n" + "result = getattr(Owner(), 'read')()\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['result']) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIs(pyobjects.get_unknown(), module['result'].get_object()) + + def test_inspect_members_preserves_selected_callable_in_heterogeneous_namespace(self): + for receiver, call in (("Owner()", "()"), ("Owner", "(Owner())")): + with self.subTest(receiver=receiver): + code = ( + "import inspect\nclass Value:\n pass\n\n" + "class Owner:\n label = 'other member'\n" + " def run(self):\n return Value()\n\n" + + "result = dict(inspect.getmembers(" + receiver + "))['run']" + call + "\n" + ) + namespace = {} + exec(code, namespace) + self.assertIsInstance(namespace['result'], namespace['Value']) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIs(module['Value'].get_object(), module['result'].get_object().get_type()) + + def test_inspect_members_dynamic_hooks_remain_unresolved(self): + code = ( + "import inspect\nclass Owner:\n" + " def __dir__(self):\n return ['run']\n" + " def run(self):\n return 7\n\n" + "result = dict(inspect.getmembers(Owner()))['run']()\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['result']) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIs(pyobjects.get_unknown(), module['result'].get_object()) + + def test_inspect_member_model_does_not_adopt_homonymous_source(self): + code = ( + "class inspect:\n @staticmethod\n def getmembers(value):\n return 'custom'\n\n" + "result = inspect.getmembers(7)\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual('custom', namespace['result']) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIsInstance(module['result'].get_object(), builtins.LiteralString) + self.assertEqual('custom', module['result'].get_object().value) + + def test_map_next_preserves_callable_element_flow(self): + code = ( + "class Value:\n pass\n\n" + "result = next(map(lambda value: value, [Value()]))\n" + ) + namespace = {} + exec(code, namespace) + self.assertIsInstance(namespace['result'], namespace['Value']) + self.mod.write(code) + module = self.project.get_pymodule(self.mod) + self.assertIs(module['Value'].get_object(), module['result'].get_object().get_type()) + def setUp(self): super().setUp() self.project = testutils.sample_project() @@ -17,6 +158,12 @@ def tearDown(self): testutils.remove_project(self.project) super().tearDown() + def test_builtin_module_exposes_scope(self): + module = builtins.BuiltinModule("builtins") + scope = module.get_scope() + self.assertEqual("Module", scope.get_kind()) + self.assertIn("len", scope.get_names()) + def test_simple_case(self): self.mod.write("l = []\n") pymod = self.project.get_pymodule(self.mod) @@ -59,6 +206,25 @@ class C(object): a_var = pymod["a_var"].get_object() self.assertEqual(c_class, a_var.get_type()) + def test_get_items_passes_the_subscript_argument(self): + code = dedent("""\ + class Value: + pass + class Container: + def __getitem__(self, item): + return item + value = Value() + selected = Container()[value] + """) + namespace = {} + exec(code, namespace) + self.assertIs(namespace["value"], namespace["selected"]) + self.mod.write(code) + pymod = self.project.get_pymodule(self.mod) + self.assertIs( + pymod["value"].get_object(), pymod["selected"].get_object() + ) + def test_get_items_from_slices(self): self.mod.write(dedent("""\ class C(object): @@ -401,6 +567,83 @@ def b_f(self): a_var = pymod["a_var"].get_object() self.assertEqual(c_class, a_var.get_type()) + def test_super_follows_the_method_resolution_order(self): + self.mod.write(dedent("""\ + class C1(object): + pass + class C2(object): + pass + class Base(object): + def a_f(self): + return C1() + class Left(Base): + pass + class Right(Base): + def a_f(self): + return C2() + class Derived(Left, Right): + def b_f(self): + return super(Derived, self).a_f() + a_var = Derived().b_f() + """)) + pymod = self.project.get_pymodule(self.mod) + c2_class = pymod["C2"].get_object() + a_var = pymod["a_var"].get_object() + self.assertEqual(c2_class, a_var.get_type()) + + def test_super_binds_the_receiver(self): + self.mod.write(dedent("""\ + class Base(object): + def get_self(self): + return self + class Derived(Base): + def get(self): + return super(Derived, self).get_self() + a_var = Derived().get() + """)) + pymod = self.project.get_pymodule(self.mod) + derived = pymod["Derived"].get_object() + a_var = pymod["a_var"].get_object() + self.assertEqual(derived, a_var.get_type()) + + def test_super_classmethod_binds_the_receiver_class(self): + self.mod.write(dedent("""\ + class Base(object): + @classmethod + def make(cls): + return cls() + class Derived(Base): + @classmethod + def build(cls): + return super(Derived, cls).make() + a_var = Derived.build() + """)) + pymod = self.project.get_pymodule(self.mod) + derived = pymod["Derived"].get_object() + a_var = pymod["a_var"].get_object() + self.assertEqual(derived, a_var.get_type()) + + def test_super_skips_the_class_itself(self): + self.mod.write(dedent("""\ + class C1(object): + pass + class C2(object): + pass + class Base(object): + def a_f(self): + return C1() + class Derived(Base): + def a_f(self): + return C2() + def b_f(self): + return super(Derived, self).a_f() + a_var = Derived().b_f() + """)) + pymod = self.project.get_pymodule(self.mod) + c1_class = pymod["C1"].get_object() + a_var = pymod["a_var"].get_object() + self.assertEqual(c1_class, a_var.get_type()) + def test_file_builtin_type(self): self.mod.write(dedent("""\ for line in open("file.txt"): diff --git a/ropetest/codeanalyzetest.py b/ropetest/codeanalyzetest.py index debbfd169..bc13b0bb0 100644 --- a/ropetest/codeanalyzetest.py +++ b/ropetest/codeanalyzetest.py @@ -386,6 +386,56 @@ def test_is_assigned_here_for_not_equal_test(self): word_finder = worder.Worder(code) self.assertFalse(word_finder.is_assigned_here(0)) + def test_subscript_assignment_does_not_assign_its_index(self): + for code in ( + "get_registry()[name] = value\n", + "registry[name]=value\n", + "registry[name]+=value\n", + "registry[(name)]=value\n", + "registry[left + name] = value\n", + "registry[:name] = value\n", + "registry[name:]=value\n", + "registry[registry[name]]=value\n", + "[registry[name]]=values\n", + ): + with self.subTest(code=code): + word_finder = worder.Worder(code) + self.assertFalse(word_finder.is_assigned_here(code.index("name"))) + + def test_grouped_and_list_assignment_targets_are_assigned(self): + for code in ( + "(name)=value\n", + "(name) = value\n", + "[name]=value\n", + "[name] = value\n", + "[(name)]=value\n", + "([name])=value\n", + ): + with self.subTest(code=code): + word_finder = worder.Worder(code) + self.assertTrue(word_finder.is_assigned_here(code.index("name"))) + + def test_assignment_operator_types(self): + for operator in ( + "=", "+=", "-=", "*=", "/=", "//=", "%=", "**=", "@=", + "&=", "|=", "^=", "<<=", ">>=", ":=", + ): + code = "name " + operator + " value\n" + if operator == ":=": + code = "(" + code.rstrip() + ")\n" + with self.subTest(operator=operator): + word_finder = worder.Worder(code) + self.assertEqual( + operator, word_finder.get_assignment_type(code.index("name")) + ) + + def test_comparison_operators_do_not_assign(self): + for operator in ("==", "!=", "<=", ">="): + code = "name " + operator + " value\n" + with self.subTest(operator=operator): + word_finder = worder.Worder(code) + self.assertFalse(word_finder.is_assigned_here(0)) + # XXX: is_assigned_here should work for tuple assignments def xxx_test_is_assigned_here_for_tuple_assignment(self): code = "a, b = (1, 2)\n" @@ -669,6 +719,29 @@ class C(object): result = name_finder.get_pyname_at(len(code) - 12) self.assertEqual(a_var_pyname, result) + def test_source_binding_index_reads_module_all_literal_sequence(self): + module = libutils.get_string_module( + self.project, '__all__ = ("Public", "helper")\n' + ) + + self.assertEqual( + ("Public", "helper"), + module.get_source_binding_index().assignment_string_sequence("__all__"), + ) + + def test_source_binding_index_reads_class_metadata_all_literal_sequence(self): + module = libutils.get_string_module( + self.project, + 'class Public:\n __all__ = ["Public", "helper"]\n', + ) + + self.assertEqual( + ("Public", "helper"), + module.get_source_binding_index().assignment_string_sequence( + "__all__", class_metadata=True + ), + ) + def test_class_method_attribute_in_class_body(self): code = dedent("""\ class C(object): diff --git a/ropetest/conftest.py b/ropetest/conftest.py index 99ec1b76b..0d618788f 100644 --- a/ropetest/conftest.py +++ b/ropetest/conftest.py @@ -1,3 +1,4 @@ +import multiprocessing import os import pathlib from subprocess import check_call @@ -9,6 +10,11 @@ from ropetest import testutils +@pytest.hookimpl(tryfirst=True) +def pytest_sessionstart(session): + multiprocessing.set_start_method("spawn") + + @pytest.fixture(scope="session") def session_venv(tmpdir_factory): path = tmpdir_factory.mktemp("venv") @@ -28,10 +34,10 @@ def session_venv_pyvenv_cfg(session_venv): @pytest.fixture(scope="session") def session_venv_site_packages(session_venv, session_venv_pyvenv_cfg): - if os.name == 'nt': - return session_venv / f"Lib/site-packages" + if os.name == "nt": + return session_venv / "Lib/site-packages" else: - major, minor, patch = session_venv_pyvenv_cfg["version"].split(".") + major, minor, _patch = session_venv_pyvenv_cfg["version"].split(".") return session_venv / f"lib/python{major}.{minor}/site-packages" @@ -95,13 +101,15 @@ def mod2(project, pkg1) -> resources.Folder: @pytest.fixture(scope="session") def external_fixturepkg(session_venv, session_venv_python_executable): - check_call([ - session_venv_python_executable, - "-m", - "pip", - "install", - "--force-reinstall", - "ropetest-package-fixtures/external_fixturepkg/dist/external_fixturepkg-1.0.0-py3-none-any.whl", - ]) + check_call( + [ + session_venv_python_executable, + "-m", + "pip", + "install", + "--force-reinstall", + "ropetest-package-fixtures/external_fixturepkg/dist/external_fixturepkg-1.0.0-py3-none-any.whl", + ] + ) yield check_call([session_venv_python_executable, "-m", "pip", "uninstall", "--yes", "external-fixturepkg"]) diff --git a/ropetest/contrib/autoimport/autoimporttest.py b/ropetest/contrib/autoimport/autoimporttest.py index 072ab1ed8..3d0b664ce 100644 --- a/ropetest/contrib/autoimport/autoimporttest.py +++ b/ropetest/contrib/autoimport/autoimporttest.py @@ -1,4 +1,7 @@ +import multiprocessing import sqlite3 +import threading +import warnings from concurrent.futures import ThreadPoolExecutor from contextlib import closing, contextmanager from textwrap import dedent @@ -6,6 +9,7 @@ import pytest +from rope.base import exceptions from rope.base.project import Project from rope.base.resources import File, Folder from rope.contrib.autoimport import models @@ -14,7 +18,7 @@ @pytest.fixture def autoimport(project: Project): - with closing(AutoImport(project)) as ai: + with closing(AutoImport(project, memory=True)) as ai: yield ai @@ -29,33 +33,33 @@ def database_list(connection): def test_in_memory_database_share_cache(project, project2): - ai_1 = AutoImport(project, memory=True) - ai_2 = AutoImport(project, memory=True) - - ai_3 = AutoImport(project2, memory=True) - - with ai_1.connection: - ai_1.connection.execute("CREATE TABLE shared(data)") - ai_1.connection.execute("INSERT INTO shared VALUES(28)") - assert ai_2.connection.execute("SELECT data FROM shared").fetchone() == (28,) - with pytest.raises(sqlite3.OperationalError, match="no such table: shared"): - ai_3.connection.execute("SELECT data FROM shared").fetchone() + with ( + AutoImport(project, memory=True) as ai_1, + AutoImport(project, memory=True) as ai_2, + AutoImport(project2, memory=True) as ai_3, + ): + with ai_1.connection: + ai_1.connection.execute("CREATE TABLE shared(data)") + ai_1.connection.execute("INSERT INTO shared VALUES(28)") + assert ai_2.connection.execute("SELECT data FROM shared").fetchone() == (28,) + with pytest.raises(sqlite3.OperationalError, match="no such table: shared"): + ai_3.connection.execute("SELECT data FROM shared").fetchone() def test_autoimport_connection_parameter_with_in_memory( project: Project, autoimport: AutoImport, ): - connection = AutoImport.create_database_connection(memory=True) - assert is_in_memory_database(connection) + with closing(AutoImport.create_database_connection(memory=True)) as connection: + assert is_in_memory_database(connection) def test_autoimport_connection_parameter_with_project( project: Project, autoimport: AutoImport, ): - connection = AutoImport.create_database_connection(project=project) - assert not is_in_memory_database(connection) + with closing(AutoImport.create_database_connection(project=project)) as connection: + assert not is_in_memory_database(connection) def test_autoimport_create_database_connection_conflicting_parameter( @@ -70,16 +74,26 @@ def test_autoimport_memory_parameter_is_true( project: Project, autoimport: AutoImport, ): - ai = AutoImport(project, memory=True) - assert is_in_memory_database(ai.connection) + with AutoImport(project, memory=True) as ai: + assert is_in_memory_database(ai.connection) + + +def test_autoimport_omitted_memory_parameter_is_deprecated(project: Project): + with pytest.warns( + DeprecationWarning, + match=r"The default value for `AutoImport\(memory\)` argument will change", + ): + with AutoImport(project) as ai: + assert ai.memory is True + assert is_in_memory_database(ai.connection) def test_autoimport_memory_parameter_is_false( project: Project, autoimport: AutoImport, ): - ai = AutoImport(project, memory=False) - assert not is_in_memory_database(ai.connection) + with AutoImport(project, memory=False) as ai: + assert not is_in_memory_database(ai.connection) def test_init_py( @@ -101,8 +115,44 @@ def foo(): assert [("from pkg1 import foo", "foo")] == results -def test_multithreading( +@pytest.fixture +def fork_default_start_method(): + # The suite selects "spawn" globally; the cache workers must not depend + # on that, so this restores the POSIX default "fork" for the test. + if "fork" not in multiprocessing.get_all_start_methods(): + pytest.skip("the platform has no fork start method") + previous = multiprocessing.get_start_method(allow_none=True) + multiprocessing.set_start_method("fork", force=True) + yield + multiprocessing.set_start_method(previous, force=True) + + +@pytest.mark.usefixtures("fork_default_start_method") +def test_cache_workers_do_not_fork_a_threaded_process( autoimport: AutoImport, + pkg1: Folder, +): + # Forking a process that runs other threads can deadlock the child. + # CPython reports that warning from inside fork(), where it cannot be + # raised, so the test records warnings instead of relying on the + # "error" filter. + pkg1.get_child("__init__.py").write("def foo():\n pass\n") + stop = threading.Event() + helper = threading.Thread(target=stop.wait) + helper.start() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + autoimport.generate_cache([pkg1.get_child("__init__.py")]) + autoimport.generate_modules_cache(["textwrap"]) + finally: + stop.set() + helper.join() + assert [str(warning.message) for warning in caught] == [] + assert ("from pkg1 import foo", "foo") in autoimport.search("foo", True) + + +def test_multithreading( project: Project, pkg1: Folder, mod1: File, @@ -114,22 +164,129 @@ def foo(): """)) mod1.write(dedent("""\ foo - """)) - autoimport = AutoImport(project, memory=False) - autoimport.generate_cache([mod1_init]) + """) + ) + with closing(AutoImport(project, memory=False)) as autoimport: + autoimport.generate_cache([mod1_init]) + + with ThreadPoolExecutor(1) as tp: + results = tp.submit(autoimport.search, "foo", True).result() + assert [("from pkg1 import foo", "foo")] == results + + +def test_multithread_connections_closed_on_close(project: Project): + with AutoImport(project, memory=True) as ai: + main_conn = ai.connection + worker_conns = [] + + def worker(): + conn = ai.connection + worker_conns.append(conn) + return list(ai.search("foo")) + + with ThreadPoolExecutor(3) as tp: + futures = [tp.submit(worker) for _ in range(3)] + for f in futures: + f.result() + + all_conns = {main_conn} | set(worker_conns) + assert len(all_conns) > 1 + assert all_conns.issubset(ai._connections) + + assert ai._closed + assert len(ai._connections) == 0 + for conn in all_conns: + with pytest.raises(sqlite3.ProgrammingError, match="Cannot operate on a closed database"): + conn.execute("SELECT 1") + + with pytest.raises(exceptions.RopeError, match="AutoImport instance has been closed"): + _ = ai.connection + + +def test_close_thread_connection(project: Project): + with AutoImport(project, memory=True) as ai: + worker_conn = None + + def worker(): + nonlocal worker_conn + worker_conn = ai.connection + assert worker_conn in ai._connections + ai.close_thread_connection() + assert worker_conn not in ai._connections + with pytest.raises(sqlite3.ProgrammingError, match="Cannot operate on a closed database"): + worker_conn.execute("SELECT 1") + new_conn = ai.connection + assert new_conn is not worker_conn + assert new_conn in ai._connections + + with ThreadPoolExecutor(1) as tp: + tp.submit(worker).result() + + +def test_close_idempotent(project: Project): + ai = AutoImport(project, memory=True) + conn = ai.connection + ai.close() + assert ai._closed + ai.close() + with pytest.raises(sqlite3.ProgrammingError, match="Cannot operate on a closed database"): + conn.execute("SELECT 1") - tp = ThreadPoolExecutor(1) - results = tp.submit(autoimport.search, "foo", True).result() - assert [("from pkg1 import foo", "foo")] == results + +def test_register_connection_after_close(project: Project): + ai = AutoImport(project, memory=True) + ai.close() + conn = AutoImport.create_database_connection(memory=True) + with pytest.raises(exceptions.RopeError, match="AutoImport instance has been closed"): + ai._register_connection(conn) + assert conn not in ai._connections + with pytest.raises(sqlite3.ProgrammingError, match="Cannot operate on a closed database"): + conn.execute("SELECT 1") + + +def test_connection_setter_after_close(project: Project): + ai = AutoImport(project, memory=True) + ai.close() + with closing(AutoImport.create_database_connection(memory=True)) as conn: + with pytest.raises(exceptions.RopeError, match="AutoImport instance has been closed"): + ai.connection = conn + + +def test_connection_setter_replaces_existing(project: Project): + with AutoImport(project, memory=True) as ai, closing(ai.connection) as old_conn: + assert old_conn in ai._connections + with closing(AutoImport.create_database_connection(memory=True)) as conn: + ai.connection = conn + assert ai.connection is conn + assert conn in ai._connections + assert old_conn not in ai._connections def test_connection(project: Project, project2: Project): - ai1 = AutoImport(project) - ai2 = AutoImport(project) - ai3 = AutoImport(project2) + with ( + AutoImport(project, memory=True) as ai1, + AutoImport(project, memory=True) as ai2, + AutoImport(project2, memory=True) as ai3, + ): + assert ai1.connection is not ai2.connection + assert ai1.connection is not ai3.connection + + +def test_resource_to_module_skips_non_package_folders( + autoimport: AutoImport, + project: Project, +): + # Intermediate folders that are not packages (no ``__init__.py``) must not + # be included in the derived module name. + # Regression test for https://github.com/python-rope/rope/issues/823 + src = project.root.create_folder("src") + pkg = src.create_folder("mypkg") + pkg.create_file("__init__.py") + documents = pkg.create_file("documents.py") + + module = autoimport._resource_to_module(documents) - assert ai1.connection is not ai2.connection - assert ai1.connection is not ai3.connection + assert module.modname == "mypkg.documents" @contextmanager diff --git a/ropetest/contrib/autoimport/conftest.py b/ropetest/contrib/autoimport/conftest.py index 9dca8ccf5..6c1d0d7dc 100644 --- a/ropetest/contrib/autoimport/conftest.py +++ b/ropetest/contrib/autoimport/conftest.py @@ -1,4 +1,7 @@ +import importlib.machinery +import importlib.util import pathlib +import sys import pytest @@ -43,6 +46,17 @@ def example_external_package_path( @pytest.fixture def compiled_lib(): - import _sqlite3 - - return "_sqlite3", pathlib.Path(_sqlite3.__file__) + """Return one stdlib extension module that this interpreter loads from a file. + + Which modules are shared libraries depends on how the interpreter was + built (python-build-standalone links _sqlite3 statically, so it has no + ``__file__``), so the module is discovered from the import system instead + of being hard-coded. + """ + for name in sorted(sys.stdlib_module_names): + spec = importlib.util.find_spec(name) + if spec is not None and isinstance( + spec.loader, importlib.machinery.ExtensionFileLoader + ): + return name, pathlib.Path(spec.origin) + pytest.fail("this interpreter loads no stdlib extension module from a file") diff --git a/ropetest/contrib/autoimport/modeltest.py b/ropetest/contrib/autoimport/modeltest.py index 0044434cc..8eef06d35 100644 --- a/ropetest/contrib/autoimport/modeltest.py +++ b/ropetest/contrib/autoimport/modeltest.py @@ -8,7 +8,9 @@ @pytest.fixture def empty_db(): - return sqlite3.connect(":memory:") + conn = sqlite3.connect(":memory:") + yield conn + conn.close() class TestQuery: diff --git a/ropetest/contrib/autoimport/utilstest.py b/ropetest/contrib/autoimport/utilstest.py index d807debe2..3ecb0bd0c 100644 --- a/ropetest/contrib/autoimport/utilstest.py +++ b/ropetest/contrib/autoimport/utilstest.py @@ -1,9 +1,14 @@ """Tests for autoimport utility functions, written in pytest""" -from pathlib import Path +import sys +import venv +from pathlib import Path, PurePosixPath, PureWindowsPath + +import pytest from rope.contrib.autoimport import utils from rope.contrib.autoimport.defs import Package, PackageType, Source +from rope.contrib.autoimport.sqlite import AutoImport def test_get_package_source(mod1_path, project): @@ -14,11 +19,6 @@ def test_get_package_source_not_project(mod1_path): assert utils.get_package_source(mod1_path, None, "") == Source.UNKNOWN -def test_get_package_source_project_prefix_is_not_ancestry(project): - sibling = Path(project.address + "-other") / "module.py" - assert utils.get_package_source(sibling, project, "module") == Source.UNKNOWN - - def test_get_package_source_project_sibling(project): sibling = project.root.pathlib.with_name(project.root.pathlib.name + "-other") sibling.mkdir() @@ -33,6 +33,31 @@ def test_get_package_source_project_sibling(project): sibling.rmdir() +def test_get_package_source_ignored_nested_resource(project): + folder = project.root.pathlib / "ignored" / "nested" + folder.mkdir(parents=True) + module = folder / "outside.py" + module.touch() + project.prefs["ignored_resources"] += ["ignored/*"] + assert utils.get_package_source(module, project, "outside") == Source.UNKNOWN + + +def test_get_package_source_precedence(project): + folder = project.root.pathlib / "site-packages" + folder.mkdir() + module = folder / "typing.py" + module.touch() + assert utils.get_package_source(module, project, "typing") == Source.PROJECT + assert utils.get_package_source(module, None, "typing") == Source.SITE_PACKAGE + builtin = sys.builtin_module_names[0] + assert utils.get_package_source(module, project, builtin) == Source.BUILTIN + + +def test_get_package_source_project_prefix_is_not_ancestry(project): + sibling = Path(project.address + "-other") / "module.py" + assert utils.get_package_source(sibling, project, "module") == Source.UNKNOWN + + def test_get_package_source_pytest(example_external_package_path): # pytest is not installed as part of the standard library # but should be installed into site_packages, @@ -41,6 +66,29 @@ def test_get_package_source_pytest(example_external_package_path): assert source == Source.SITE_PACKAGE +def test_get_package_source_venv_in_project_dir(example_external_package_path, project): + # Many Python dev tools create .venv folders inside the project directory. + # Modules in such folders should count as SITE_PACKAGE files despite + # technically being inside the project folder. + + # Set up actual venv in project folder: + project_venv_path = project.root.pathlib/".venv" + venv.create(project_venv_path) + + # Crude approximation of a package installed into this venv: + project_venv_site_packages_path = next(project_venv_path.glob("**/*site-packages")) + project.prefs["python_path"].append(project_venv_site_packages_path) + module_path = project_venv_site_packages_path / "foo.py" + module_path.touch() + + # Such directories are normally part of `ignored_resources` (e.g. `.venv` + # is in there by default): + project.prefs["ignored_resources"] += [".venv"] + + source = utils.get_package_source(module_path, project, "foo") + assert source == Source.SITE_PACKAGE + + def test_get_package_source_typing(typing_path): assert utils.get_package_source(typing_path, None, "typing") == Source.STANDARD @@ -53,6 +101,23 @@ def test_get_modname_single_file(typing_path): assert utils.get_modname_from_path(typing_path, typing_path) == "typing" +@pytest.mark.parametrize("path_type", [PurePosixPath, PureWindowsPath]) +def test_get_modname_path_components(path_type): + root = path_type("runtime/Lib") + assert utils.get_modname_from_path(root / "email" / "message.py", root) == ( + "Lib.email.message" + ) + with pytest.raises(RuntimeError, match="No site-packages allowed here"): + utils.get_modname_from_path(root / "site-packages" / "cfgv.py", root) + + +def test_get_modname_external(example_external_package_path): + assert utils.get_modname_from_path( + example_external_package_path, + example_external_package_path, + ) == "external_fixturepkg" + + def test_get_modname_folder( example_external_package_path, example_external_package_module_path, @@ -92,3 +157,53 @@ def test_get_files(project, mod1, pkg1, mod2): Path("pkg1/__init__.py"), Path("pkg1/mod2.py"), } + + +@pytest.mark.parametrize("underlined", [False, True]) +def test_get_files_separate_site_packages(tmp_path, project, underlined): + library = tmp_path / "runtime" / "Lib" + site_packages = library / "site-packages" + distribution = site_packages / "fixture_distribution" + distribution.mkdir(parents=True) + (library / "stdlib_fixture.py").write_text("stdlib_name = 1\n") + (site_packages / "cfgv.py").write_text("cfgv_name = 1\n") + (distribution / "__init__.py").write_text("distribution_name = 1\n") + (distribution / "module.py").write_text("module_name = 1\n") + + package = utils.get_package_tuple(library) + assert package.source == Source.UNKNOWN + assert {module.modname for module in utils.get_files(package, underlined)} == { + "Lib.stdlib_fixture" + } + external = utils.get_package_tuple(distribution) + assert external.source == Source.SITE_PACKAGE + assert {module.modname for module in utils.get_files(external, underlined)} == { + "fixture_distribution", + "fixture_distribution.module", + } + + project.prefs["python_path"].extend([str(library.parent), str(site_packages)]) + with AutoImport(project, observe=False, memory=True) as autoimport: + autoimport.generate_modules_cache( + ["Lib", "cfgv", "fixture_distribution"], + single_thread=True, + underlined=underlined, + ) + assert autoimport.get_modules("stdlib_name") == ["Lib.stdlib_fixture"] + assert autoimport.get_modules("cfgv_name") == ["cfgv"] + assert autoimport.get_modules("distribution_name") == ["fixture_distribution"] + assert autoimport.get_modules("module_name") == ["fixture_distribution.module"] + + +def test_get_files_keeps_nested_installation_roots_separate(tmp_path): + library = tmp_path / "Lib" + installed = library / "site-packages" + installed.mkdir(parents=True) + (library / "stdlib_module.py").write_text("value = 1\n") + (installed / "dependency.py").write_text("value = 2\n") + package = utils.get_package_tuple(library) + assert [module.modname for module in utils.get_files(package)] == [ + "Lib.stdlib_module" + ] + dependency = utils.get_package_tuple(installed / "dependency.py") + assert [module.modname for module in utils.get_files(dependency)] == ["dependency"] diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index d42ec14f7..51773c482 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -12,9 +12,10 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=False) + self.importer = autoimport.AutoImport(self.project, observe=False, memory=True) def tearDown(self): + self.importer.close() testutils.remove_project(self.project) super().tearDown() @@ -87,10 +88,12 @@ def test_not_caching_underlined_names(self): self.assertEqual(["mod1"], self.importer.get_modules("_myvar")) def test_caching_underlined_names_passing_to_the_constructor(self): - importer = autoimport.AutoImport(self.project, False, True) - self.mod1.write("_myvar = None\n") - importer.update_resource(self.mod1) - self.assertEqual(["mod1"], importer.get_modules("_myvar")) + with autoimport.AutoImport( + self.project, observe=False, underlined=True, memory=True + ) as importer: + self.mod1.write("_myvar = None\n") + importer.update_resource(self.mod1) + self.assertEqual(["mod1"], importer.get_modules("_myvar")) def test_name_locations(self): self.mod1.write("myvar = None\n") @@ -121,10 +124,12 @@ def test_search_module(self): def test_search_alias(self): self.mod2.write("myvar = None\n") self.importer.update_resource(self.mod2) - self.importer.add_aliases([ - ("noMatch", "does_not_exists_this"), - ("hasMatch", "pkg.mod2"), - ]) + self.importer.add_aliases( + [ + ("noMatch", "does_not_exists_this"), + ("hasMatch", "pkg.mod2"), + ] + ) self.assertEqual([], self.importer.search("noMatch", exact_match=True)) @@ -178,12 +183,12 @@ def test_skipping_directories_not_accessible_because_of_permission_error(self): def test_search_submodule(project, external_fixturepkg): - importer = autoimport.AutoImport(project, observe=False) - importer.update_module("external_fixturepkg") - import_statement = ("from external_fixturepkg import mod1", "mod1") - assert import_statement in importer.search("mod1", exact_match=True) - assert import_statement in importer.search("mo") - assert import_statement in importer.search("mod1") + with autoimport.AutoImport(project, observe=False, memory=True) as importer: + importer.update_module("external_fixturepkg") + import_statement = ("from external_fixturepkg import mod1", "mod1") + assert import_statement in importer.search("mod1", exact_match=True) + assert import_statement in importer.search("mo") + assert import_statement in importer.search("mod1") class AutoImportObservingTest(unittest.TestCase): @@ -193,9 +198,10 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=True) + self.importer = autoimport.AutoImport(self.project, observe=True, memory=True) def tearDown(self): + self.importer.close() testutils.remove_project(self.project) super().tearDown() diff --git a/ropetest/contrib/findittest.py b/ropetest/contrib/findittest.py index db4887598..58f9ed1a2 100644 --- a/ropetest/contrib/findittest.py +++ b/ropetest/contrib/findittest.py @@ -3,6 +3,7 @@ from rope.base import exceptions from rope.contrib.findit import find_definition, find_implementations, find_occurrences +from rope.refactor.rename import Rename from ropetest import testutils @@ -49,6 +50,15 @@ def f(arg): ) self.assertEqual(2, len(result)) + def test_find_occurrences_after_the_module_changes(self): + mod = testutils.create_module(self.project, "mod") + mod.write("a_var = 1\nprint(a_var)\n") + result = find_occurrences(self.project, mod, 1) + self.assertEqual([0, 16], [occurrence.offset for occurrence in result]) + mod.write("\na_var = 1\nb_var = a_var\nprint(a_var)\n") + result = find_occurrences(self.project, mod, 2) + self.assertEqual([1, 19, 31], [occurrence.offset for occurrence in result]) + def test_find_occurrences_resources_parameter(self): mod1 = testutils.create_module(self.project, "mod1") mod2 = testutils.create_module(self.project, "mod2") @@ -149,6 +159,63 @@ async def run(): self.assertIsNotNone(result) self.assertEqual(def_offset, result.offset) + # A class-method def following an f-string with an unmatched literal + # bracket: the unmatched bracket used to fold every following newline, + # desyncing offsets so worder mis-read the method def header -- + # find_definition returned None (or the call site) and Rename silently + # corrupted the file (renamed the call, orphaned the def). Module-level + # defs after the same f-string stayed immune. + + _FSTRING_METHOD_CODE = dedent("""\ + class C(object): + def run(self): + label = f"[{self}" + self.target() + + def target(self): + pass + """) + + @testutils.only_for_versions_higher("3.12") + def test_find_definition_of_method_after_fstring_bracket(self): + code = self._FSTRING_METHOD_CODE + call_offset = code.index("self.target()") + len("self.") + result = find_definition(self.project, code, call_offset) + def_offset = code.index("def target") + len("def ") + self.assertIsNotNone(result) + self.assertEqual(def_offset, result.offset) + + @testutils.only_for_versions_higher("3.12") + def test_rename_heals_method_after_fstring_bracket(self): + mod = testutils.create_module(self.project, "mod") + code = self._FSTRING_METHOD_CODE + mod.write(code) + call_offset = code.index("self.target()") + len("self.") + changes = Rename(self.project, mod, call_offset).get_changes("renamed") + self.project.do(changes) + result = mod.read() + # both the call and the def must be renamed -- not silently orphaned + self.assertEqual(0, result.count("target")) + self.assertEqual(2, result.count("renamed")) + + @testutils.only_for_versions_higher("3.12") + def test_module_function_after_fstring_still_resolves(self): + # module-level defs after the same f-string are immune and must stay working + code = dedent("""\ + def run(): + label = f"[{run}" + + def target(): + pass + + target() + """) + call_offset = code.rindex("target") + result = find_definition(self.project, code, call_offset) + def_offset = code.index("def target") + len("def ") + self.assertIsNotNone(result) + self.assertEqual(def_offset, result.offset) + def test_find_definition_instance_method(self): code = dedent("""\ class C(object): @@ -223,3 +290,5 @@ class C(object): def_offset = code.index("target") self.assertIsNotNone(result) self.assertEqual(def_offset, result.offset) + self.assertIsNotNone(result) + self.assertEqual(def_offset, result.offset) diff --git a/ropetest/external_data_attributes_test.py b/ropetest/external_data_attributes_test.py new file mode 100644 index 000000000..f1e5256ad --- /dev/null +++ b/ropetest/external_data_attributes_test.py @@ -0,0 +1,1436 @@ +import re +import subprocess +import sys +import textwrap + +import pytest + +from rope.base import evaluate, exceptions, pyobjects +from rope.base.project import Project +from rope.refactor.rename import Rename + +CLASSES = textwrap.dedent("""class First: + def __init__(self): + self.field = 1 +class Other: + def __init__(self): + self.field = 2 +""") +BOX = textwrap.dedent("""class Box: + def __init__(self): + self.field = 0 +""") + + +def driver(code, directory): + compile(code, "subject.py", "exec") + result = subprocess.run( + [sys.executable, "-B", "-c", code], + cwd=directory, + capture_output=True, + text=True, + timeout=20, + ) + return {"exit": result.returncode, "stdout": result.stdout, "stderr": result.stderr} + + +def exercise( + tmp_path, + code, + field, + expected, + reject=False, + warm=False, + replacement=None, + uncertain_classes=("Box",), +): + directory = tmp_path + record = { + "source": code, + "expected": expected, + "field": field, + "expected_rejection": reject, + } + project = Project( + str(directory), + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + ) + try: + resource = project.root.create_file("subject.py") + resource.write(code) + record["before"] = driver(code, directory) + assert record["before"]["exit"] == 0 and record["before"]["stdout"] == expected + + def inspect(): + module = project.get_pymodule(resource) + finder = evaluate.ScopeNameFinder(module) + rows = [] + # Late-first queries stress the same cache without inventing values. + for match in reversed(list(re.finditer(r"\.field\b", resource.read()))): + pyname = finder.get_pyname_at(match.start() + 2) + rows.append( + { + "offset": match.start() + 2, + "none": pyname is None, + "definition": ( + pyname.get_definition_location()[1] if pyname else None + ), + } + ) + for class_name in ["Box", "Base", "Child"]: + pyname = module.get_scope().lookup(class_name) + if pyname is not None: + obj = pyname.get_object() + if ( + isinstance(obj, pyobjects.AbstractClass) + and "slot" in obj.get_attributes() + ): + slot = obj.get_attribute("slot") + rows.append( + { + "class": class_name, + "slot_unknown": slot.get_object() + is pyobjects.get_unknown(), + "slot_definition": slot.get_definition_location()[1], + } + ) + return rows + + if warm: + record["warm_queries"] = inspect() + record["hot_queries"] = inspect() + if replacement is not None: + resource.write(replacement) + code = replacement + record["replacement_source"] = code + record["replacement_before"] = driver(code, directory) + assert record["replacement_before"]["exit"] == 0 + before_bytes = resource.read_bytes() + offset = ( + code.index(field) + 1 + if isinstance(field, str) + else code.index(f"self.field = {field}") + len("self.") + 1 + ) + record["selected_offset"] = offset + try: + changes = Rename(project, resource, offset).get_changes("changed") + except exceptions.RefactoringError as error: + record["refactor_error"] = str(error) + record["atomic_unchanged"] = resource.read_bytes() == before_bytes + assert reject, error + assert record["atomic_unchanged"] + record["after_source"] = resource.read() + record["after"] = driver(record["after_source"], directory) + else: + project.do(changes) + record["after_source"] = resource.read() + record["after"] = driver(record["after_source"], directory) + if reject: + pytest.fail( + "Ambiguous data field Rename was accepted instead of atomically rejected" + ) + expected_after = record.get("replacement_before", record["before"]) + assert ( + record["after"]["exit"] == 0 + and record["after"]["stdout"] == expected_after["stdout"] + ), record["after"] + record["final_queries"] = inspect() + if reject: + assert all( + row["slot_unknown"] + for row in record["final_queries"] + if row.get("class") in uncertain_classes + ) + finally: + project.close() + + +@pytest.mark.parametrize("read", ["Box.slot", "Box().slot"]) +@pytest.mark.parametrize("placeholder", [False, True]) +@pytest.mark.parametrize("field", [1, 2]) +def test_single_external_data_field_rename(tmp_path, read, placeholder, field): + box = BOX + (" slot = None\n" if placeholder else "") + code = ( + CLASSES + + box + + textwrap.dedent(f"Box.slot = Other()\nresult = {read}\nprint(result.field)\n") + ) + exercise(tmp_path, code, field, "2\n") + + +@pytest.mark.parametrize("field", [1, 2]) +def test_rhs_uses_module_namespace_and_stable_data_variable(tmp_path, field): + code = ( + CLASSES + + BOX + + textwrap.dedent( + " Other = First\n slot = None\npeer = Other()\nBox.slot = peer\nresult = Box().slot\nprint(result.field)\n" + ) + ) + exercise(tmp_path, code, field, "2\n", warm=True) + + +def test_unrelated_same_named_attribute_does_not_poison_supported_data(tmp_path): + code = ( + CLASSES + + textwrap.dedent("class Unrelated:\n slot = First()\n") + + BOX + + textwrap.dedent( + "print(Unrelated.slot.field)\nBox.slot = Other()\nprint(Box().slot.field)\n" + ) + ) + exercise(tmp_path, code, 2, textwrap.dedent("1\n2\n"), warm=True) + + +def test_resource_write_invalidates_external_data_value_and_canonical_fields(tmp_path): + code = ( + CLASSES + + BOX + + textwrap.dedent( + "Box.slot = Other()\nresult = Box().slot\nprint(result.field)\n" + ) + ) + replacement = code.replace("Box.slot = Other()", "Box.slot = First()") + exercise(tmp_path, code, 1, "2\n", warm=True, replacement=replacement) + + +@pytest.mark.parametrize("saved", [False, True]) +@pytest.mark.parametrize("field", [1, 2]) +def test_early_read_and_saved_value_are_unknown_and_atomic(tmp_path, saved, field): + box = BOX + " slot = First()\n" + early = ( + textwrap.dedent("saved = Box().slot\nprint(saved.field)\n") + if saved + else "print(Box.slot.field)\n" + ) + code = ( + CLASSES + + box + + early + + textwrap.dedent("Box.slot = Other()\nprint(Box().slot.field)\n") + ) + exercise(tmp_path, code, field, textwrap.dedent("1\n2\n"), reject=True, warm=True) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_repeated_data_writes_are_unknown_and_atomic(tmp_path, field): + code = ( + CLASSES + + BOX + + textwrap.dedent( + "Box.slot = First()\nBox.slot = Other()\nprint(Box.slot.field)\n" + ) + ) + exercise(tmp_path, code, field, "2\n", reject=True) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_conditional_data_only_blocks_affected_field(tmp_path, field): + code = ( + CLASSES + + BOX + + textwrap.dedent("if True:\n Box.slot = Other()\nprint(Box().slot.field)\n") + ) + exercise(tmp_path, code, field, "2\n", reject=field == 2, warm=True) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_rebound_class_and_saved_class_alias_are_unknown_and_atomic(tmp_path, field): + code = ( + CLASSES + + BOX + + textwrap.dedent( + " slot = First()\nclass Replacement:\n slot = First()\nold = Box\nBox = Replacement\nBox.slot = Other()\nprint(old.slot.field)\nprint(Box.slot.field)\n" + ) + ) + exercise(tmp_path, code, field, textwrap.dedent("1\n2\n"), reject=True, warm=True) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_inheritance_does_not_merge_base_and_child_values(tmp_path, field): + code = CLASSES + textwrap.dedent( + "class Base:\n slot = None\nclass Child(Base):\n slot = First()\nBase.slot = Other()\nprint(Child.slot.field)\nprint(Base.slot.field)\n" + ) + exercise( + tmp_path, + code, + field, + textwrap.dedent("1\n2\n"), + reject=field == 2, + warm=True, + uncertain_classes=("Base",), + ) + + +@pytest.mark.parametrize("field", [0, "marker"]) +def test_uncertain_data_does_not_block_unrelated_class_or_global_rename( + tmp_path, field +): + code = ( + "marker = 42\n" + + CLASSES + + BOX + + textwrap.dedent( + "if True:\n Box.slot = Other()\nprint(marker)\nprint(Box().slot.field)\n" + ) + ) + exercise(tmp_path, code, field, textwrap.dedent("42\n2\n"), warm=True) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_instance_attribute_and_class_data_remain_distinct_or_unknown(tmp_path, field): + code = CLASSES + textwrap.dedent( + "class Box:\n def __init__(self):\n self.field = 0\n self.slot = First()\nBox.slot = Other()\nprint(Box().slot.field)\nprint(Box.slot.field)\n" + ) + exercise(tmp_path, code, field, textwrap.dedent("1\n2\n"), reject=True, warm=True) + + +def test_deferred_data_read_is_unknown_and_atomic(tmp_path): + code = ( + CLASSES + + BOX + + textwrap.dedent( + "def read():\n return Box.slot\nBox.slot = Other()\nprint(read().field)\n" + ) + ) + exercise(tmp_path, code, 2, "2\n", reject=True, warm=True) + + +@pytest.mark.parametrize( + "rhs", + [ + "callback", + "lambda self, item: self", + "staticmethod(callback)", + "classmethod(callback)", + ], +) +def test_ordinary_callable_negative_control_retains_legacy_boundary(tmp_path, rhs): + if rhs == "staticmethod(callback)": + definition = textwrap.dedent("def callback(item):\n return item\n") + expected, field = "2\n", 1 + elif rhs == "classmethod(callback)": + definition = textwrap.dedent("def callback(cls, item):\n return item\n") + expected, field = "2\n", 1 + else: + definition = textwrap.dedent("def callback(self, item):\n return self\n") + expected, field = "0\n", 2 + code = ( + CLASSES + + definition + + BOX + + textwrap.dedent( + f"Box.callback = {rhs}\nresult = Box().callback(Other())\nprint(result.field)\n" + ) + ) + exercise(tmp_path, code, field, expected, warm=True) + + +def test_custom_descriptor_negative_control_is_not_new_data_support(tmp_path): + code = ( + CLASSES + + textwrap.dedent( + "class Descriptor:\n def __get__(self, instance, owner):\n return Other()\n" + ) + + BOX + + textwrap.dedent("Box.callback = Descriptor()\nprint(Box().callback.field)\n") + ) + exercise(tmp_path, code, 1, "2\n", warm=True) + + +@pytest.mark.parametrize("warm", [False, True]) +def test_new_public_query_canonical_identity_and_resource_refresh(tmp_path, warm): + directory = tmp_path + code = ( + CLASSES + + BOX + + textwrap.dedent( + "Box.slot = Other()\nresult = Box().slot\nprint(result.field)\n" + ) + ) + project = Project( + str(directory), + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + ) + record = {"source": code} + try: + resource = project.root.create_file("subject.py") + resource.write(code) + record["before"] = driver(code, directory) + assert record["before"]["exit"] == 0 and record["before"]["stdout"] == "2\n" + + def query(expected): + module = project.get_pymodule(resource) + scope = module.get_scope() + data_class = scope.lookup(expected).get_object() + finder = evaluate.ScopeNameFinder(module) + field = data_class.get_attribute("field") + result = scope.lookup("result").get_object() + assert result.get_type() is data_class + field_read = finder.get_pyname_at( + resource.read().index("print(result.field)") + len("print(result.") + 1 + ) + assert field_read is field + box = scope.lookup("Box").get_object() + slot = box.get_attribute("slot") + write_name = finder.get_pyname_at( + resource.read().index("Box.slot") + len("Box.") + 1 + ) + read_name = finder.get_pyname_at( + resource.read().index("Box().slot") + len("Box().") + 1 + ) + assert write_name is read_name is slot + assert slot.get_object().get_type() is data_class + assert slot.get_definition_location()[0] is module + assert ( + slot.get_definition_location()[1] + == resource.read()[: resource.read().index("Box.slot")].count("\n") + 1 + ) + return module, slot, field + + old_module, old_slot, old_field = query("Other") + if warm: + assert query("Other") == (old_module, old_slot, old_field) + replacement = code.replace("Box.slot = Other()", "Box.slot = First()") + resource.write(replacement) + new_module, new_slot, new_field = query("First") + assert ( + new_module is not old_module + and new_slot is not old_slot + and new_field is not old_field + ) + record["after_resource_write"] = driver(replacement, directory) + assert ( + record["after_resource_write"]["exit"] == 0 + and record["after_resource_write"]["stdout"] == "1\n" + ) + record["canonical_identity_passed"] = True + finally: + project.close() + + +@pytest.mark.parametrize( + "lookup", + ["holder-getattribute", "holder-getattr", "metaclass", "decorator", "rhs-new"], +) +def test_new_lookup_rewriting_boundary_is_unknown_and_atomic(tmp_path, lookup): + if lookup == "holder-getattribute": + box = BOX + textwrap.indent( + textwrap.dedent( + " def __getattribute__(self, name):\n if name == 'slot':\n return First()\n return super().__getattribute__(name)\n" + ), + " ", + ) + code = ( + CLASSES + + box + + textwrap.dedent("Box.slot = Other()\nprint(Box().slot.field)\n") + ) + field, expected = 1, "1\n" + elif lookup == "holder-getattr": + box = BOX + textwrap.indent( + textwrap.dedent( + " def __getattr__(self, name):\n return First()\n" + ), + " ", + ) + code = ( + CLASSES + + box + + textwrap.dedent("Box.slot = Other()\nprint(Box().slot.field)\n") + ) + field, expected = 2, "2\n" + elif lookup == "metaclass": + meta = textwrap.dedent( + "class Meta(type):\n def __getattribute__(cls, name):\n if name == 'slot':\n return First()\n return super().__getattribute__(name)\n" + ) + code = ( + CLASSES + + meta + + textwrap.dedent( + "class Box(metaclass=Meta):\n pass\nBox.slot = Other()\nprint(Box.slot.field)\n" + ) + ) + field, expected = 1, "1\n" + elif lookup == "decorator": + code = ( + CLASSES + + textwrap.dedent("def decorate(cls):\n return cls\n@decorate\n") + + BOX + + textwrap.dedent("Box.slot = Other()\nprint(Box().slot.field)\n") + ) + field, expected = 2, "2\n" + else: + modified = CLASSES.replace( + "class Other:\n", + textwrap.dedent( + "class Other:\n def __new__(cls):\n return First()\n" + ), + ) + code = ( + modified + + BOX + + textwrap.dedent("Box.slot = Other()\nprint(Box().slot.field)\n") + ) + field, expected = 1, "1\n" + exercise(tmp_path, code, field, expected, reject=True, warm=True) + + +@pytest.mark.parametrize("timing", ["before", "after"]) +@pytest.mark.parametrize("field", [1, 2]) +def test_global_class_rebinding_blocks_affected_fields(tmp_path, timing, field): + prefix = CLASSES + textwrap.dedent( + "class Box:\n pass\nclass Replacement:\n slot = Other()\n" + ) + rebind = textwrap.dedent( + "def rebind():\n global Box\n Box = Replacement\nrebind()\n" + ) + if timing == "before": + code = ( + prefix + + rebind + + textwrap.dedent( + "Box.slot = First()\nprint(Box.slot.field, Replacement.slot.field)\n" + ) + ) + expected = "1 1\n" + else: + code = ( + prefix + + textwrap.dedent("Box.slot = First()\nsaved = Box.slot\n") + + rebind + + "print(saved.field, Box.slot.field)\n" + ) + expected = "1 2\n" + exercise( + tmp_path, + code, + field, + expected, + reject=True, + warm=True, + uncertain_classes=("Box", "Replacement"), + ) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_class_body_lookup_alias_does_not_use_plain_data_inference(tmp_path, field): + lookup = textwrap.dedent( + "def lookup(self, name):\n if name == 'slot':\n return First()\n return object.__getattribute__(self, name)\n" + ) + code = ( + CLASSES + + lookup + + textwrap.dedent( + "class Box:\n __getattribute__ = lookup\nBox.slot = Other()\nprint(Box.slot.field, Box().slot.field)\n" + ) + ) + exercise(tmp_path, code, field, "2 1\n", reject=True, warm=True) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_module_instance_write_keeps_class_data_uncertain(tmp_path, field): + code = CLASSES + textwrap.dedent( + "class Box:\n pass\nBox.slot = Other()\nbox = Box()\nbox.slot = First()\nprint(Box.slot.field, box.slot.field)\n" + ) + exercise(tmp_path, code, field, "2 1\n", reject=True, warm=True) + + +@pytest.mark.parametrize("field", [1, 2]) +def test_lookup_factory_parameter_reassignment_does_not_lose_guard_type( + tmp_path, field +): + factory = textwrap.dedent( + "def choose(item):\n item = First()\n return item\n" + ) + box = textwrap.dedent( + "class Box:\n def __getattribute__(self, name):\n if name == 'slot':\n return choose(Other())\n return object.__getattribute__(self, name)\n" + ) + code = ( + CLASSES + + factory + + box + + textwrap.dedent( + "Box.slot = Other()\nprint(Box.slot.field, Box().slot.field)\n" + ) + ) + exercise(tmp_path, code, field, "2 1\n", reject=True, warm=True) + + +def test_lookup_keyword_only_expansion_does_not_lose_affected_data_type(tmp_path): + third = textwrap.dedent( + "class Third:\n def __init__(self):\n self.field = 3\n" + ) + choose = textwrap.dedent( + "def choose(*, item):\n if False:\n item = First()\n return item\n" + ) + box = textwrap.dedent( + "class Box:\n def __getattribute__(self, name):\n if name == 'slot':\n return choose(**{'item': Other()})\n return object.__getattribute__(self, name)\n" + ) + code = ( + CLASSES + + third + + choose + + box + + textwrap.dedent( + "Box.slot = Third()\nprint(Box.slot.field, Box().slot.field)\n" + ) + ) + exercise(tmp_path, code, 2, "3 2\n", reject=True, warm=True) + + +def exercise_imported_slot( + tmp_path, selection, *, early=None, module_import=False, reexport=False +): + values = textwrap.dedent("class Foreign:\n field = 'FOREIGN'\nold = Foreign()\n") + binding = ( + " import values as slot\n" + if module_import + else " from values import old as slot\n" + ) + subject = ( + textwrap.dedent("class Other:\n field = 'OTHER'\nclass Owner:\n") + binding + ) + if early == "class-body": + subject += " saved = slot\n" + elif early == "module": + subject += "saved = Owner.slot\n" + subject += "Owner.slot = Other()\n" + if early == "class-body": + subject += "print(Owner.saved.field)\n" + elif early == "module": + subject += ( + "print(saved.old.field)\n" if module_import else "print(saved.field)\n" + ) + else: + subject += "print(Owner.slot.field)\n" + expected = "FOREIGN\n" if early else "OTHER\n" + project = Project( + str(tmp_path), + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + ) + try: + resources = { + name: project.root.create_file(name) for name in ["values.py", "subject.py"] + } + if reexport: + resources["foreign.py"] = project.root.create_file("foreign.py") + resources["foreign.py"].write(values.removesuffix("old = Foreign()\n")) + resources["values.py"].write( + textwrap.dedent("from foreign import Foreign\nold = Foreign()\n") + ) + else: + resources["values.py"].write(values) + resources["subject.py"].write(subject) + before_sources = { + name: resource.read_bytes() for name, resource in resources.items() + } + + def runtime(): + result = subprocess.run( + [sys.executable, "-B", "-c", "import subject"], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=20, + ) + return result.returncode, result.stdout, result.stderr + + before = runtime() + assert before[0] == 0 and before[1] == expected, before + resource = resources["subject.py"] + if selection == "other": + offset = subject.index("field = 'OTHER'") + changed_name = "changed_field" + elif selection == "foreign": + resource = resources["foreign.py"] if reexport else resources["values.py"] + offset = values.index("field = 'FOREIGN'") + changed_name = "changed_field" + elif selection == "slot": + offset = subject.index("Owner.slot =") + len("Owner.") + changed_name = "changed_slot" + elif selection == "dependency": + resource = resources["values.py"] + offset = values.index("old =") + changed_name = "changed_original" + else: + resource = resources["values.py"] + offset = None + changed_name = "changed_values" + try: + changes = Rename(project, resource, offset).get_changes(changed_name) + except exceptions.RefactoringError: + assert early, "Unambiguous imported-slot overwrite must be refactorable" + assert { + name: value.read_bytes() for name, value in resources.items() + } == before_sources + else: + project.do(changes) + changed_subject = resources["subject.py"].read() + if selection in ["other", "foreign"]: + assert f"changed_field = '{selection.upper()}'" in resource.read() + elif selection == "slot": + assert "Owner.changed_slot = Other()" in changed_subject + assert "as changed_slot" in changed_subject + elif selection == "dependency": + assert "changed_original = Foreign()" in resource.read() + assert "import changed_original as slot" in changed_subject + else: + assert not resources["values.py"].exists() + assert project.get_file("changed_values.py").exists() + assert "import changed_values as slot" in changed_subject + after = runtime() + assert after[0] == 0 and after[1] == expected, after + if not early: + source = resources["subject.py"].read() + module = project.get_pymodule(resources["subject.py"]) + other = module.get_scope().lookup("Other").get_object() + field_name = "changed_field" if selection == "other" else "field" + slot_name = "changed_slot" if selection == "slot" else "slot" + finder = evaluate.ScopeNameFinder(module) + field = finder.get_pyname_at( + source.index(f"Owner.{slot_name}.{field_name}") + + len(f"Owner.{slot_name}.") + ) + assert field is other.get_attribute(field_name) + write = finder.get_pyname_at( + source.index(f"Owner.{slot_name} =") + len("Owner.") + ) + read = finder.get_pyname_at( + source.index(f"Owner.{slot_name}.{field_name}") + len("Owner.") + ) + assert ( + write + is read + is module.get_scope() + .lookup("Owner") + .get_object() + .get_attribute(slot_name) + ) + assert write.get_object().get_type() is other + finally: + project.close() + + +@pytest.mark.parametrize("selection", ["other", "foreign", "slot", "dependency"]) +def test_safe_imported_slot_overwrite_preserves_source_and_new_binding( + tmp_path, selection +): + exercise_imported_slot(tmp_path, selection) + + +@pytest.mark.parametrize("selection", ["other", "module"]) +def test_safe_imported_module_slot_uses_new_value_and_keeps_module_source( + tmp_path, selection +): + exercise_imported_slot(tmp_path, selection, module_import=True) + + +@pytest.mark.parametrize("early", ["module", "class-body"]) +def test_early_imported_foreign_field_is_preserved_or_atomically_refused( + tmp_path, early +): + exercise_imported_slot(tmp_path, "foreign", early=early) + + +def test_early_imported_module_keeps_instance_field_or_atomically_refused(tmp_path): + exercise_imported_slot(tmp_path, "foreign", early="module", module_import=True) + + +def test_early_reexported_module_keeps_instance_field_or_atomically_refused(tmp_path): + exercise_imported_slot( + tmp_path, "foreign", early="module", module_import=True, reexport=True + ) + + +def test_early_imported_package_keeps_foreign_field_or_atomically_refuses(tmp_path): + sources = { + "foreign.py": textwrap.dedent("class Foreign:\n field = 'FOREIGN'\n"), + "valuespkg/__init__.py": textwrap.dedent( + "from foreign import Foreign\nold = Foreign()\n" + ), + "subject.py": textwrap.dedent( + "class Other:\n field = 'OTHER'\nclass Owner:\n import valuespkg as slot\nsaved = Owner.slot\nOwner.slot = Other()\nprint(saved.old.field)\n" + ), + } + project = Project( + str(tmp_path), + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + ) + try: + package = project.root.create_folder("valuespkg") + resources = { + "foreign.py": project.root.create_file("foreign.py"), + "valuespkg/__init__.py": package.create_file("__init__.py"), + "subject.py": project.root.create_file("subject.py"), + } + for name, resource in resources.items(): + resource.write(sources[name]) + original = {name: resource.read_bytes() for name, resource in resources.items()} + before = driver("import subject", tmp_path) + assert before["exit"] == 0 and before["stdout"] == "FOREIGN\n", before + try: + changes = Rename( + project, resources["foreign.py"], sources["foreign.py"].index("field =") + ).get_changes("changed_field") + except exceptions.RefactoringError: + assert { + name: resource.read_bytes() for name, resource in resources.items() + } == original + else: + project.do(changes) + assert "changed_field = 'FOREIGN'" in resources["foreign.py"].read() + after = driver("import subject", tmp_path) + assert after["exit"] == 0 and after["stdout"] == "FOREIGN\n", after + finally: + project.close() + + +# Module export and Unicode import regressions, using existing test helpers. + + +def _module_data_project(directory, sources): + project = Project( + str(directory), + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + ) + resources = {} + for path, code in sources.items(): + parent = project.root + parts = path.split("/") + for part in parts[:-1]: + parent = ( + parent.get_child(part) + if parent.has_child(part) + else parent.create_folder(part) + ) + resources[path] = parent.create_file(parts[-1]) + resources[path].write(code) + project.root.create_file("sentinel.py").write("untouched = 'sentinel'\n") + return project, resources + + +def _python_source_bytes(directory): + return { + path.relative_to(directory).as_posix(): path.read_bytes() + for path in directory.rglob("*.py") + } + + +def _preserve_saved_foreign_field( + tmp_path, sources, reference, target="foreign-field", expect_success=False +): + sources = dict(sources) + if target == "unrelated-global": + sources["subject.py"] = "unconnected = 'OK'\n" + sources["subject.py"].replace( + f"print({reference})", f"print({reference}, unconnected)" + ) + expected = "FOREIGN OK\n" if target == "unrelated-global" else "FOREIGN\n" + project, resources = _module_data_project(tmp_path, sources) + try: + before = driver("import subject", tmp_path) + assert before["exit"] == 0 and before["stdout"] == expected, before + original = _python_source_bytes(tmp_path) + if target == "unrelated-global": + selected_path, marker, new_name = ( + "subject.py", + "unconnected = 'OK'", + "changed_unconnected", + ) + else: + selected_path = ( + "foreign.py" if "foreign.py" in sources else "valuespkg/foreign.py" + ) + marker, new_name = "field = 'FOREIGN'", "renamed_field" + resource = resources[selected_path] + offset = sources[selected_path].index(marker) + try: + changes = Rename(project, resource, offset).get_changes(new_name) + except exceptions.RefactoringError: + assert target == "foreign-field", "Unrelated global Rename must succeed" + assert not expect_success, "Ordinary referenced field Rename must succeed" + assert _python_source_bytes(tmp_path) == original + else: + # get_changes itself must leave every resource unchanged. + assert _python_source_bytes(tmp_path) == original + project.do(changes) + if target == "unrelated-global": + assert "changed_unconnected = 'OK'" in resource.read() + assert f"print({reference}, changed_unconnected)" in resource.read() + current = _python_source_bytes(tmp_path) + assert { + key: value for key, value in current.items() if key != "subject.py" + } == { + key: value for key, value in original.items() if key != "subject.py" + } + else: + assert "renamed_field = 'FOREIGN'" in resource.read() + assert ( + reference.replace(".field", ".renamed_field") + in resources["subject.py"].read() + ) + assert "field = 'OTHER'" in resources["subject.py"].read() + assert project.get_file("sentinel.py").read_bytes() == original["sentinel.py"] + after = driver("import subject", tmp_path) + assert after["exit"] == 0 and after["stdout"] == expected, after + finally: + project.close() + + +_MODULE_DATA_SUBJECT = textwrap.dedent("""\ + class Other: + field = 'OTHER' + class Owner: + import values as slot + saved = Owner.slot + Owner.slot = Other() + print(saved.old.field) + """) + + +@pytest.mark.parametrize( + "case", ["factory-result", "module-alias", "legal-cycle", "nested-class"] +) +@pytest.mark.parametrize("target", ["foreign-field", "unrelated-global"]) +def test_saved_imported_module_data_keeps_field_safety_and_unrelated_rename( + tmp_path, case, target +): + subject = _MODULE_DATA_SUBJECT + reference = "saved.old.field" + if case == "factory-result": + sources = { + "foreign.py": textwrap.dedent("""\ + class Foreign: + field = 'FOREIGN' + def make(): + return Foreign() + """), + "values.py": textwrap.dedent("""\ + from foreign import make + old = make() + """), + "subject.py": subject, + } + elif case == "module-alias": + reference = "saved.alias.old.field" + sources = { + "foreign.py": textwrap.dedent("""\ + class Foreign: + field = 'FOREIGN' + old = Foreign() + """), + "values.py": "import foreign as alias\n", + "subject.py": subject.replace("saved.old.field", reference), + } + elif case == "legal-cycle": + sources = { + "foreign.py": textwrap.dedent("""\ + import values as values_namespace + class Foreign: + field = 'FOREIGN' + old = Foreign() + """), + "values.py": textwrap.dedent("""\ + import foreign as foreign_namespace + old = foreign_namespace.old + """), + "subject.py": subject, + } + else: + reference = "saved.Inner.field" + sources = { + "foreign.py": textwrap.dedent("""\ + class Outer: + class Inner: + field = 'FOREIGN' + old = Outer() + """), + "subject.py": textwrap.dedent("""\ + class Other: + field = 'OTHER' + class Owner: + from foreign import old as slot + saved = Owner.slot + Owner.slot = Other() + print(saved.Inner.field) + """), + } + _preserve_saved_foreign_field(tmp_path, sources, reference, target) + + +_IMPORT_SPAN_VALUES = textwrap.dedent("""\ + class Foreign: + field = 'FOREIGN' + old = Foreign() + """) +_IMPORT_SPAN_SUBJECT = textwrap.dedent("""\ + from pkg.values import old as foreign + class Other: + field = 'OTHER' + class 容器: + 标签 = '中'; import pkg.values as pkg + 容器.pkg = Other() + print(foreign.field, 容器.pkg.field) + """) + + +@pytest.mark.parametrize( + "kind, identifier, new_name", + [ + ("package", "pkg", "renamed_pkg"), + ("module", "values", "renamed_values"), + ("alias", "pkg", "renamed_slot"), + ], +) +def test_dotted_import_source_and_local_alias_rename_with_unicode_prefix( + tmp_path, kind, identifier, new_name +): + sources = { + "pkg/__init__.py": "", + "pkg/values.py": _IMPORT_SPAN_VALUES, + "subject.py": _IMPORT_SPAN_SUBJECT, + } + project, resources = _module_data_project(tmp_path, sources) + try: + before = driver("import subject", tmp_path) + assert before["exit"] == 0 and before["stdout"] == "FOREIGN OTHER\n", before + original = _python_source_bytes(tmp_path) + source = sources["subject.py"] + line = next( + value + for value in source.splitlines() + if "import pkg.values as pkg" in value + ) + character_column = line.index("import pkg.values") + byte_column = len(line[:character_column].encode("utf-8")) + assert character_column == 14 and byte_column == 20 + marker = "容器.pkg = Other()" if kind == "alias" else "import pkg.values as pkg" + # Public offsets are Unicode character offsets, including the source word. + offset = source.index(marker) + marker.index(identifier) + changes = Rename(project, resources["subject.py"], offset).get_changes(new_name) + assert _python_source_bytes(tmp_path) == original + project.do(changes) + subject = resources["subject.py"].read() + if kind == "alias": + values_path, init_path = "pkg/values.py", "pkg/__init__.py" + expected_subject = source.replace("as pkg", "as renamed_slot").replace( + "容器.pkg", "容器.renamed_slot" + ) + slot_name = "renamed_slot" + # The source package/module spelling remains intact. + assert "import pkg.values as renamed_slot" in subject + assert "from pkg.values import old as foreign" in subject + else: + slot_name = "pkg" + target_module = ( + "renamed_pkg.values" if kind == "package" else "pkg.renamed_values" + ) + values_path = ( + "renamed_pkg/values.py" + if kind == "package" + else "pkg/renamed_values.py" + ) + init_path = ( + "renamed_pkg/__init__.py" if kind == "package" else "pkg/__init__.py" + ) + expected_subject = source.replace("pkg.values", target_module) + assert not project.get_file("pkg/values.py").exists() + if kind == "package": + assert not project.root.has_child("pkg") + assert f"import {target_module} as pkg" in subject + assert f"from {target_module} import old as foreign" in subject + assert subject == expected_subject + assert project.get_file(values_path).read() == sources["pkg/values.py"] + assert project.get_file(init_path).read() == "" + assert project.get_file("sentinel.py").read_bytes() == original["sentinel.py"] + after = driver("import subject", tmp_path) + assert after["exit"] == 0 and after["stdout"] == "FOREIGN OTHER\n", after + module = project.get_pymodule(resources["subject.py"]) + finder = evaluate.ScopeNameFinder(module) + for marker, expected_path, expected_line, class_name in ( + ("foreign.field", values_path, 2, "Foreign"), + (f"容器.{slot_name}.field", "subject.py", 3, "Other"), + ): + field_offset = subject.index(marker) + marker.rindex(".field") + 1 + pyname = finder.get_pyname_at(field_offset) + assert pyname is not None + assert pyname.get_object() is not pyobjects.get_unknown() + owner_module = project.get_pymodule(project.get_file(expected_path)) + expected_pyname = ( + owner_module.get_attribute(class_name) + .get_object() + .get_attribute("field") + ) + assert pyname is expected_pyname + defining, lineno = pyname.get_definition_location() + assert ( + defining is not None and defining.get_resource().path == expected_path + ) + assert lineno == expected_line + finally: + project.close() + + +def test_saved_namespace_module_foreign_field_is_preserved_or_atomically_refused( + tmp_path, +): + # Select a field from the real module inside a namespace package. + sources = { + "valuespkg/foreign.py": textwrap.dedent("""\ + class Foreign: + field = 'FOREIGN' + old = Foreign() + """), + "subject.py": textwrap.dedent("""\ + import valuespkg.foreign + class Other: + field = 'OTHER' + class Owner: + import valuespkg as slot + saved = Owner.slot + Owner.slot = Other() + print(saved.foreign.old.field) + """), + } + assert "valuespkg/__init__.py" not in sources + _preserve_saved_foreign_field(tmp_path, sources, "saved.foreign.old.field") + + +def test_ordinary_referenced_foreign_field_rename_succeeds_without_external_write( + tmp_path, +): + sources = { + "foreign.py": textwrap.dedent("""\ + class Foreign: + field = 'FOREIGN' + """), + "subject.py": textwrap.dedent("""\ + from foreign import Foreign + class Other: + field = 'OTHER' + saved = Foreign() + print(saved.field) + """), + } + _preserve_saved_foreign_field(tmp_path, sources, "saved.field", expect_success=True) + + +_NAMESPACE_COMMON = textwrap.dedent("""class First: + field = 'FIRST' +class Other: + field = 'OTHER' +""") + +_NAMESPACE_PATTERN = _NAMESPACE_COMMON + textwrap.dedent("""def lookup(self, name): + if name == 'slot': + return First() + return object.__getattribute__(self, name) +class Box: + match lookup: + case __getattribute__: + pass +Box.slot = Other() +print(Box.slot.field, Box().slot.field) +""") + +_NAMESPACE_TUPLE = _NAMESPACE_COMMON + textwrap.dedent("""class Box: + slot = Other() +Box.slot, spare = First(), 0 +print(Box.slot.field) +""") + +_NAMESPACE_CLASS_ALIAS = _NAMESPACE_COMMON + textwrap.dedent("""class Box: + pass +Box.slot = Other() +alias = Box +def replace(): + alias.slot = First() +replace() +print(Box.slot.field) +""") + +_NAMESPACE_WILDCARD = _NAMESPACE_COMMON + textwrap.dedent("""class Box: + pass +Box.slot = First() +saved = Box +from values import * +print(saved.slot.field, Box.slot.field) +""") + +_NAMESPACE_WILDCARD_VALUES = textwrap.dedent("""class Foreign: + field = 'FOREIGN' +class Box: + slot = Foreign() +__all__ = ['Box'] +""") + + +@pytest.mark.parametrize( + "code, values, expected", + [ + pytest.param( + _NAMESPACE_PATTERN, None, "OTHER FIRST\n", id="pattern-class-hook" + ), + pytest.param(_NAMESPACE_TUPLE, None, "FIRST\n", id="tuple-class-slot"), + pytest.param( + _NAMESPACE_CLASS_ALIAS, None, "FIRST\n", id="deferred-class-alias" + ), + pytest.param( + _NAMESPACE_WILDCARD, + _NAMESPACE_WILDCARD_VALUES, + "FIRST FOREIGN\n", + id="wildcard-class-rebinding", + ), + ], +) +def test_unsafe_namespace_writes_atomically_refuse_field_rename( + tmp_path, code, values, expected +): + project = Project( + str(tmp_path), + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + ) + try: + sources = {"subject.py": code, "sentinel.py": "untouched = 'sentinel'\n"} + if values is not None: + sources["values.py"] = values + resources = {} + for path, source in sources.items(): + resource = project.root.create_file(path) + resource.write(source) + resources[path] = resource + + before = driver("import subject", tmp_path) + assert before["exit"] == 0 and before["stdout"] == expected, before + snapshot = {path: resource.read_bytes() for path, resource in resources.items()} + + # No Rope query primes inference: Rename is the first semantic consumer. + offset = code.index("field = 'FIRST'") + 1 + with pytest.raises(exceptions.RefactoringError): + Rename(project, resources["subject.py"], offset).get_changes( + "changed_field" + ) + + assert { + path: resource.read_bytes() for path, resource in resources.items() + } == snapshot + after = driver("import subject", tmp_path) + assert after["exit"] == 0 and after["stdout"] == expected, after + finally: + project.close() + + +@pytest.mark.parametrize( + "binding", ["pattern-capture", "except-alias", "loop-unpack", "annotated-local"] +) +@pytest.mark.parametrize("field", [1, 2]) +def test_lookup_factory_local_rebinding_keeps_actual_return_type_guarded( + tmp_path, binding, field +): + classes = CLASSES + if binding == "pattern-capture": + factory = textwrap.dedent("""\ + def choose(item): + match Other(): + case item: + return item + """) + elif binding == "except-alias": + classes = CLASSES.replace("class First:", "class First(Exception):") + factory = textwrap.dedent("""\ + def choose(item): + try: + raise First() + except First as item: + return item + """) + elif binding == "loop-unpack": + factory = textwrap.dedent("""\ + def choose(item): + for item, spare in [(Other(), First())]: + return item + """) + else: + factory = textwrap.dedent("""\ + def choose(item): + item: object = Other() + return item + """) + + hook = textwrap.dedent("""\ + def __getattribute__(self, name): + if name == 'slot': + return choose(First()) + return object.__getattribute__(self, name) + """) + tail = textwrap.dedent("""\ + Box.slot = First() + print(Box.slot.field, Box().slot.field) + """) + expected = "1 2\n" + if binding == "except-alias": + hook = hook.replace("choose(First())", "choose(Other())") + tail = tail.replace("Box.slot = First()", "Box.slot = Other()") + expected = "2 1\n" + code = classes + factory + BOX + textwrap.indent(hook, " ") + tail + sentinel = tmp_path / "sentinel.py" + sentinel.write_text("untouched = 'sentinel'\n", encoding="utf-8") + original_sentinel = sentinel.read_bytes() + exercise(tmp_path, code, field, expected, reject=True, warm=False) + assert sentinel.read_bytes() == original_sentinel + + +_EXTRA_MAPPING_CLASS_IDENTITY = CLASSES + BOX + textwrap.dedent("""Box.slot = Other() +saved = Box +match {'replacement': First()}: + case {**Box}: + pass +print(saved.slot.field, Box['replacement'].field) +""") + +_EXTRA_STAR_CLASS_IDENTITY = CLASSES + BOX + textwrap.dedent("""Box.slot = Other() +saved = Box +match [First()]: + case [*Box]: + pass +print(saved.slot.field, Box[0].field) +""") + +_EXTRA_EXCEPT_CLASS_IDENTITY = CLASSES + BOX + textwrap.dedent("""Box.slot = Other() +saved = Box +try: + raise ValueError('replacement') +except ValueError as Box: + print(saved.slot.field, str(Box)) +print(saved.slot.field) +""") + +_EXTRA_EXCEPT_LOOKUP_HOOK = CLASSES + textwrap.dedent("""def lookup(self, name): + if name == 'slot': + return First() + return object.__getattribute__(self, name) +class Box: + __getattribute__ = lookup + try: + raise ValueError('replacement') + except ValueError as __getattribute__: + pass +Box.slot = Other() +print(Box.slot.field, Box().slot.field) +""") + +_EXTRA_IMPORT_ATTRIBUTE = CLASSES + textwrap.dedent("""class Box: + from values import old +Box.old = First() +print(Box.old.field) +""") + +_EXTRA_IMPORT_VALUES = textwrap.dedent("""old = object() +""") + + +@pytest.mark.parametrize( + "code, values, marker, marker_offset, replacement, expected", + [ + pytest.param( + _EXTRA_MAPPING_CLASS_IDENTITY, + None, + "field = 2", + 1, + "changed_field", + "2 1\n", + id="mapping-rest-class-identity", + ), + pytest.param( + _EXTRA_STAR_CLASS_IDENTITY, + None, + "field = 2", + 1, + "changed_field", + "2 1\n", + id="star-class-identity", + ), + pytest.param( + _EXTRA_EXCEPT_CLASS_IDENTITY, + None, + "field = 2", + 1, + "changed_field", + "2 replacement\n2\n", + id="except-class-identity", + ), + pytest.param( + _EXTRA_EXCEPT_LOOKUP_HOOK, + None, + "field = 2", + 1, + "changed_field", + "2 2\n", + id="except-clears-class-lookup-hook", + ), + pytest.param( + _EXTRA_IMPORT_ATTRIBUTE, + _EXTRA_IMPORT_VALUES, + "Box.old = First()", + len("Box.") + 1, + "changed_slot", + "1\n", + id="unaliased-import-backed-class-attribute", + ), + ], +) +def test_namespace_captures_and_import_backed_attribute_refuse_unsafe_rename( + tmp_path, code, values, marker, marker_offset, replacement, expected +): + project = Project( + str(tmp_path), + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + ) + try: + sources = {"subject.py": code, "sentinel.py": "untouched = 'sentinel'\n"} + if values is not None: + sources["values.py"] = values + resources = {} + for path, source in sources.items(): + resource = project.root.create_file(path) + resource.write(source) + resources[path] = resource + + before = driver("import subject", tmp_path) + assert before["exit"] == 0 and before["stdout"] == expected, before + snapshot = {path: resource.read_bytes() for path, resource in resources.items()} + assert code.count(marker) == 1 + + with pytest.raises(exceptions.RefactoringError): + Rename( + project, resources["subject.py"], code.index(marker) + marker_offset + ).get_changes(replacement) + + assert { + path: resource.read_bytes() for path, resource in resources.items() + } == snapshot + after = driver("import subject", tmp_path) + assert after["exit"] == 0 and after["stdout"] == expected, after + finally: + project.close() diff --git a/ropetest/keyword_labelstest.py b/ropetest/keyword_labelstest.py new file mode 100644 index 000000000..f1a98413d --- /dev/null +++ b/ropetest/keyword_labelstest.py @@ -0,0 +1,190 @@ +from textwrap import dedent + +import pytest + +from rope.base import evaluate, exceptions, pynames +from rope.refactor.inline import create_inline +from rope.refactor.rename import Rename +from ropetest import testutils + + +@pytest.fixture +def project(): + project = testutils.sample_project() + yield project + testutils.remove_project(project) + + +@pytest.fixture +def module(project): + return testutils.create_module(project, "sample") + + +@pytest.fixture( + params=[ + ( + dedent("""\ + callback = eval("lambda target: target") + """), + "callback", + ), + ( + dedent("""\ + callbacks = {"active": eval("lambda target: target")} + """), + 'callbacks["active"]', + ), + ( + dedent("""\ + class Holder: + pass + holder = Holder() + holder.callback = eval("lambda target: target") + """), + "holder.callback", + ), + ( + dedent("""\ + def get_callback(): + return eval("lambda target: target") + """), + "get_callback()", + ), + ] +) +def unknown_call(request): + setup, callee = request.param + return dedent("""\ + target = int + """) + setup + f"result = {callee}(target=target)\n" + + +def run(source): + namespace = {} + exec(source, namespace) + return namespace["result"] + + +def test_unknown_keyword_has_no_variable_identity(project, module, unknown_call): + module.write(unknown_call) + pymodule = project.get_pymodule(module) + label_offset = unknown_call.index("target=target") + assert evaluate.eval_location2(pymodule, label_offset) == (None, None) + assert ( + evaluate.eval_location(pymodule, label_offset + len("target=")) + is pymodule["target"] + ) + + +def test_rename_variable_preserves_unknown_keyword(project, module, unknown_call): + assert run(unknown_call) is int + module.write(unknown_call) + changes = Rename(project, module, unknown_call.index("target")).get_changes( + "renamed" + ) + project.do(changes) + refactored = module.read() + assert "(target=renamed)" in refactored + assert run(refactored) is int + + +def test_inline_variable_preserves_unknown_keyword(project, module, unknown_call): + assert run(unknown_call) is int + module.write(unknown_call) + changes = create_inline(project, module, unknown_call.index("target")).get_changes() + project.do(changes) + refactored = module.read() + assert "(target=int)" in refactored + assert run(refactored) is int + + +def test_unknown_keyword_cannot_select_same_named_variable(project, module): + source = dedent("""\ + target = int + callback = eval("lambda target: target") + result = callback(target=target) + """) + module.write(source) + with pytest.raises(exceptions.RefactoringError): + Rename(project, module, source.index("target=target")) + assert module.read() == source + + +@pytest.mark.parametrize( + "callee, setup", + [ + ( + "callback", + dedent("""\ + def callback(target=int): + return target + """), + ), + ( + "callback", + dedent("""\ + class Callback: + def __call__(self, target=int): + return target + callback = Callback() + """), + ), + ], +) +@pytest.mark.parametrize("selection", ["formal", "keyword"]) +def test_known_parameter_rename_preserves_identity( + project, module, callee, setup, selection +): + source = dedent("""\ + target = str + """) + setup + f"result = {callee}(target=target)\n" + module.write(source) + formal_offset = source.index("target=int") + keyword_offset = source.index("target=target") + pymodule = project.get_pymodule(module) + formal = evaluate.eval_location(pymodule, formal_offset) + assert isinstance(formal, pynames.ParameterName) + assert evaluate.eval_location(pymodule, keyword_offset) is formal + offset = formal_offset if selection == "formal" else keyword_offset + assert run(source) is str + project.do(Rename(project, module, offset).get_changes("renamed")) + refactored = module.read() + assert "renamed=int" in refactored + assert "(renamed=target)" in refactored + assert run(refactored) is str + + +def test_rename_default_value_and_keyword_argument_value(project, module): + source = dedent("""\ + target = int + def callback(value=target): + return value + result = callback(value=target) + """) + expected = dedent("""\ + renamed = int + def callback(value=renamed): + return value + result = callback(value=renamed) + """) + assert run(source) is int + module.write(source) + project.do(Rename(project, module, source.index("target")).get_changes("renamed")) + assert module.read() == expected + assert run(module.read()) is int + + +def test_builtin_keyword_retains_parameter_identity(project, module): + source = dedent("""\ + target = int + result = dict(target=target) + """) + module.write(source) + pymodule = project.get_pymodule(module) + label_offset = source.index("target=target") + assert isinstance( + evaluate.eval_location(pymodule, label_offset), pynames.ParameterName + ) + project.do(Rename(project, module, source.index("target")).get_changes("renamed")) + assert "dict(target=renamed)" in module.read() + assert run(module.read()) == {"target": int} diff --git a/ropetest/objectinfertest.py b/ropetest/objectinfertest.py index a46a7c018..3104de40d 100644 --- a/ropetest/objectinfertest.py +++ b/ropetest/objectinfertest.py @@ -2,11 +2,65 @@ from textwrap import dedent import rope.base.builtins # Use fully-qualified names for clarity. -from rope.base import libutils +from rope.base import libutils, pyobjects +from rope.refactor import occurrences from ropetest import testutils class ObjectInferTest(unittest.TestCase): + def test_formatted_string_type_preserves_nested_call_references(self): + code = "def render():\n return 7\n\nresult = f'{render()!r:>4}'\n" + namespace = {} + exec(code, namespace) + scope = libutils.get_string_scope(self.project, code) + self.assertIs(type(namespace["result"]), str) + self.assertIs( + scope["result"].get_object().get_type(), rope.base.builtins.get_str_type() + ) + references = occurrences.create_finder( + self.project, "render", scope["render"] + ).find_occurrences(pymodule=scope.pyobject) + self.assertEqual(2, len(tuple(references))) + + def test_conditional_callable_does_not_select_last_branch_identity(self): + for enabled in (True, False): + with self.subTest(enabled=enabled): + code = f"enabled = {enabled}\nexecute = eval if enabled else str\nresult = execute('7')\n" + namespace = {} + exec(code, namespace) + self.assertEqual(7 if enabled else '7', namespace["result"]) + scope = libutils.get_string_scope(self.project, code) + self.assertIs(pyobjects.get_unknown(), scope["execute"].get_object()) + + def test_conditional_callable_preserves_identical_binding(self): + code = "enabled = True\nexecute = str if enabled else str\nresult = execute(7)\n" + namespace = {} + exec(code, namespace) + self.assertEqual('7', namespace["result"]) + scope = libutils.get_string_scope(self.project, code) + self.assertIs(scope["str"].get_object(), scope["execute"].get_object()) + + def test_conditional_method_keeps_receiver_identity_in_the_proof(self): + code = ( + "class Owner:\n def run(self):\n return self\n\n" + "left = Owner()\nright = Owner()\nenabled = True\n" + "execute = left.run if enabled else right.run\nresult = execute()\n" + ) + namespace = {} + exec(code, namespace) + self.assertIs(namespace['left'], namespace['result']) + scope = libutils.get_string_scope(self.project, code) + self.assertIs(pyobjects.get_unknown(), scope['execute'].get_object()) + + def test_literal_condition_selects_only_the_proven_branch(self): + for condition in (True, False): + with self.subTest(condition=condition): + code = f"execute = eval if {condition} else str\nresult = execute('7')\n" + namespace = {} + exec(code, namespace) + scope = libutils.get_string_scope(self.project, code) + self.assertIs(scope['eval' if condition else 'str'].get_object(), scope['execute'].get_object()) + def setUp(self): super().setUp() self.project = testutils.sample_project() @@ -209,6 +263,28 @@ def p_get(self): a_var = pymod["a_var"].get_object() self.assertEqual(a_class, a_var.get_type()) + def test_getting_property_decorator_attributes_on_file_modules(self): + # Recording the getter call information must not resolve the getter + # back from its own textual form: for a decorated property the class + # attribute is the evaluated property itself, and resolving it while + # the property is still being inferred discards the inference result. + mod = testutils.create_module(self.project, "mod") + mod.write( + dedent("""\ + class A(object): + pass + class B(object): + @property + def p(self): + return A() + a_var = B().p + """) + ) + pymod = self.project.get_pymodule(mod) + a_class = pymod["A"].get_object() + a_var = pymod["a_var"].get_object() + self.assertEqual(a_class, a_var.get_type()) + def test_lambda_functions(self): code = dedent("""\ class C(object): @@ -324,6 +400,17 @@ class C(object): a_var = mod["a_var"].get_object() self.assertEqual(c_class, a_var.get_type()) + @testutils.only_for_versions_higher("3.10") + def test_match_capture_is_unknown(self): + code = dedent("""\ + captured = 1 + match value: + case [1, *_] as captured: + pass + """) + scope = libutils.get_string_scope(self.project, code) + self.assertEqual(pyobjects.get_unknown(), scope["captured"].get_object()) + def test_basic_list_comprehensions(self): code = dedent("""\ class C(object): diff --git a/ropetest/package_artifact_contract.py b/ropetest/package_artifact_contract.py new file mode 100644 index 000000000..92b125649 --- /dev/null +++ b/ropetest/package_artifact_contract.py @@ -0,0 +1,114 @@ +"""Verify built markers and the installed wheel's closed filesystem contract.""" + +import argparse +import hashlib +import os +import shutil +import subprocess +import sys +import sysconfig +import tarfile +import tempfile +import venv +import zipfile +from importlib.metadata import version +from pathlib import Path + +import toml + + +def installed_contract(expected_version, contract): + import rope + + origin = Path(rope.__file__).resolve() + installed = Path(sysconfig.get_path("purelib")).resolve() + if not origin.is_relative_to(installed): + raise RuntimeError(f"rope did not load from the installed wheel: {origin}") + if version("rope") != expected_version: + raise RuntimeError("installed rope version differs from the built artifact") + print(f"Installed rope {version('rope')} from {origin}", flush=True) + subprocess.run( + [ + sys.executable, + "-I", + "-m", + "pytest", + "-v", + "-ra", + "-c", + str(contract.parent / "pyproject.toml"), + "--rootdir", + str(contract.parent), + "--confcutdir", + str(contract.parent), + str(contract), + ], + cwd=contract.parent, + check=True, + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--installed-contract", action="store_true") + parser.add_argument("--expected-version") + parser.add_argument("--contract", type=Path) + arguments = parser.parse_args() + if arguments.installed_contract: + if arguments.expected_version is None or arguments.contract is None: + parser.error("installed contract requires its version and contract path") + installed_contract(arguments.expected_version, arguments.contract) + return + source_root = Path(__file__).resolve().parents[1] + release = toml.load(source_root / "pyproject.toml")["project"]["version"] + wheel = Path("dist") / f"rope-{release}-py3-none-any.whl" + sdist = Path("dist") / f"rope-{release}.tar.gz" + for artifact in (wheel, sdist): + print(f"SHA256 {artifact} {hashlib.sha256(artifact.read_bytes()).hexdigest()}", flush=True) + with zipfile.ZipFile(wheel) as artifact: + assert artifact.read("rope/py.typed") == b"" + with tarfile.open(sdist) as artifact: + marker = artifact.extractfile(f"rope-{release}/rope/py.typed") + assert marker is not None + with marker: + assert marker.read() == b"" + print(f"Verified rope/py.typed in {wheel} and {sdist}") + environment = Path(sys.prefix).resolve() + if environment == Path(sys.base_prefix).resolve() or environment.is_relative_to(source_root): + raise RuntimeError("artifact validation requires an exclusive environment outside the checkout") + with tempfile.TemporaryDirectory( + prefix="rope-artifact-runtime-", dir=environment.parent + ) as directory: + runtime = Path(directory) + target = runtime / "venv" + venv.EnvBuilder(with_pip=True).create(target) + python = target / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + consumer_environment = { + name: value + for name, value in os.environ.items() + if not name.startswith("PYTEST_") + } + consumer_environment["PIP_CACHE_DIR"] = str(runtime / "pip-cache") + subprocess.run( + [str(python), "-I", "-m", "pip", "install", f"{wheel.resolve()}[dev]"], + cwd=runtime, + env=consumer_environment, + check=True, + ) + contract = runtime / "snapshot_resource_contracttest.py" + shutil.copyfile(source_root / "ropetest" / contract.name, contract) + shutil.copyfile(source_root / "pyproject.toml", runtime / "pyproject.toml") + subprocess.run( + [ + str(python), "-I", str(Path(__file__).resolve()), + "--installed-contract", "--expected-version", release, + "--contract", str(contract), + ], + cwd=runtime, + env=consumer_environment, + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/ropetest/parameter_kinds_signature_test.py b/ropetest/parameter_kinds_signature_test.py new file mode 100644 index 000000000..d97f1d08d --- /dev/null +++ b/ropetest/parameter_kinds_signature_test.py @@ -0,0 +1,355 @@ +"""Runtime and atomicity checks for parameter-kind signature consumers.""" + +import contextlib +import io +from textwrap import dedent + +import pytest + +from rope.base import exceptions +from rope.base.project import Project +from rope.refactor import change_signature, introduce_parameter, move, usefunction +from rope.refactor.inline import InlineVariable, create_inline +from rope.refactor.localtofield import LocalToField +from rope.refactor.method_object import MethodObject + + +@pytest.fixture +def project(tmp_path): + result = Project( + str(tmp_path), save_objectdb=False, save_history=False, automatic_soa=False + ) + yield result + result.close() + + +def module(project, source, name="case"): + result = project.root.create_file(name + ".py") + result.write(source) + return result + + +def execute(source): + output = io.StringIO() + with contextlib.redirect_stdout(output): + exec(compile(source, "", "exec"), {}) + return output.getvalue() + + +def refused(resources, operation): + sources = [resource.read() for resource in resources] + outputs = [execute(source) for source in sources] + try: + with pytest.raises(exceptions.RefactoringError): + operation() + finally: + assert [resource.read() for resource in resources] == sources + assert [execute(resource.read()) for resource in resources] == outputs + + +@pytest.mark.parametrize( + "parameters", ["value, /", "*, value", "value=1, *, required=2"] +) +@pytest.mark.parametrize( + "api", ["get_changes", "normalize", "remove", "add", "inline_default", "reorder"] +) +def test_signature_writers_refuse_new_kinds_atomically(project, parameters, api): + source = dedent(f"""\ + def f({parameters}): + return value + """) + resource = module(project, source) + other = module(project, "print('unrelated')\n", "other") + signature = change_signature.ChangeSignature(project, resource, source.index("f(")) + operations = { + "get_changes": lambda: signature.get_changes( + [change_signature.ArgumentNormalizer()] + ), + "normalize": signature.normalize, + "remove": lambda: signature.remove(0), + "add": lambda: signature.add(0, "added", "0", "0"), + "inline_default": lambda: signature.inline_default(0), + "reorder": lambda: signature.reorder([0]), + } + if api == "get_changes": + refused([resource, other], operations[api]) + else: + with pytest.warns(DeprecationWarning, match=rf"^{api} is deprecated$") as recorded: + refused([resource, other], operations[api]) + assert len(recorded) == 1 + + +@pytest.mark.parametrize("kind", ["posonly", "kwonly"]) +def test_signature_hierarchy_checks_actual_override(project, kind): + parameters = "self, value, /" if kind == "posonly" else "self, *, value" + source = ( + dedent("""\ + class Base: + def f(self, value): + return value + """) + + dedent(f"""\ + class Child(Base): + def f({parameters}): + return value + 1 + """) + ) + source += ( + "print(Child().f(value=3))\n" if kind == "kwonly" else "print(Child().f(3))\n" + ) + resource = module(project, source) + other = module(project, "print('unrelated')\n", "other") + signature = change_signature.ChangeSignature(project, resource, source.index("f(")) + refused( + [resource, other], + lambda: signature.get_changes( + [change_signature.ArgumentNormalizer()], in_hierarchy=True + ), + ) + + +@pytest.mark.parametrize("selection", ["class", "initializer", "call"]) +def test_constructor_signature_refuses_posonly(project, selection): + source = dedent("""\ + class C: + def __init__(self, /, value): + self.value = value + item = C(3) + print(item.value) + """) + resource = module(project, source) + offsets = { + "class": source.index("C:"), + "initializer": source.index("__init__"), + "call": source.index("C(3)"), + } + signature = change_signature.ChangeSignature(project, resource, offsets[selection]) + refused( + [resource], + lambda: signature.get_changes([change_signature.ArgumentNormalizer()]), + ) + + +@pytest.mark.parametrize( + "header", ["def f(value, /)", "def f(*, value)", "async def f(value, /)"] +) +def test_introduce_parameter_refuses_header_rewrite(project, header): + source = dedent(f"""\ + base = 3 + {header}: + return value + base + """) + resource = module(project, source) + operation = introduce_parameter.IntroduceParameter( + project, resource, source.rindex("base") + 1 + ) + refused([resource], lambda: operation.get_changes("added")) + + +@pytest.mark.parametrize( + "parameters", ["value=1, *, required", "value=1, /", "*values", "**values"] +) +def test_parameter_inline_refuses_unsupported_whole_signature(project, parameters): + source = dedent(f"""\ + def f({parameters}): + pass + """) + resource = module(project, source) + name = "values" if "values" in parameters else "value" + refused( + [resource], + lambda: create_inline(project, resource, source.index(name) + 1).get_changes(), + ) + + +def test_direct_variable_inliner_refuses_parameter(project): + source = dedent("""\ + def f(value): + return value + print(f(3)) + """) + resource = module(project, source) + refused( + [resource], lambda: InlineVariable(project, resource, source.index("value") + 1) + ) + + +@pytest.mark.parametrize("parameters", ["self, value, /", "self, *, value"]) +@pytest.mark.parametrize("direct", [False, True]) +def test_move_method_refuses_unsupported_header(project, parameters, direct): + source = dedent(f"""\ + class Target: + pass + class Owner: + def __init__(self): + self.dest = Target() + def act({parameters}): + return value + 1 + """) + resource = module(project, source) + operation = move.MoveMethod(project, resource, source.index("act(")) + refused( + [resource], + lambda: ( + operation.get_new_method("act") if direct else operation.get_changes("dest") + ), + ) + + +def test_use_function_refuses_kwonly_generated_calls(project): + source = dedent("""\ + def f(*, value): + return value + 1 + answer = 3 + 1 + print(answer) + """) + resource = module(project, source) + operation = usefunction.UseFunction(project, resource, source.index("f(")) + refused([resource], operation.get_changes) + + +def test_ordinary_signature_normalization_keeps_runtime(project): + source = dedent("""\ + def f(value): + return value + 1 + print(f(value=3)) + """) + resource = module(project, source) + signature = change_signature.ChangeSignature(project, resource, source.index("f(")) + project.do(signature.get_changes([change_signature.ArgumentNormalizer()])) + assert execute(source) == execute(resource.read()) == "4\n" + + +def test_ordinary_parameter_default_inline_keeps_runtime(project): + source = dedent("""\ + def f(value=1): + return value + 1 + print(f()) + """) + resource = module(project, source) + project.do( + create_inline(project, resource, source.index("value") + 1).get_changes() + ) + assert "f(1)" in resource.read() + assert execute(source) == execute(resource.read()) == "2\n" + + +def test_posonly_method_inline_keeps_existing_success(project): + source = dedent("""\ + def f(value, /, other): + return value + other + 1 + answer = f(1, 2) + print(answer) + """) + resource = module(project, source) + project.do(create_inline(project, resource, source.index("f(")).get_changes()) + assert execute(source) == execute(resource.read()) == "4\n" + + +def test_posonly_method_inline_refuses_invalid_keyword_call(project): + source = dedent("""\ + def f(value, /): + return value + try: + print(f(value=3)) + except TypeError: + print("keyword rejected") + """) + resource = module(project, source) + refused( + [resource], + lambda: create_inline(project, resource, source.index("f(")).get_changes(), + ) + + +def test_kwonly_method_inline_keeps_existing_refusal(project): + source = dedent("""\ + def f(value, *, other): + return value + other + print(f(1, other=2)) + """) + resource = module(project, source) + refused( + [resource], + lambda: create_inline(project, resource, source.index("f(")).get_changes(), + ) + + +@pytest.mark.parametrize( + "parameters, call", [("value, /", "3"), ("*, value", "value=3")] +) +def test_method_object_preserves_original_parameter_contract(project, parameters, call): + source = dedent(f"""\ + def f({parameters}): + return value + 1 + print(f({call})) + """) + resource = module(project, source) + operation = MethodObject(project, resource, source.index("f(")) + project.do(operation.get_changes(classname="Callable")) + assert f"def f({parameters}):" in resource.read() + assert execute(source) == execute(resource.read()) == "4\n" + + +def test_local_to_field_preserves_posonly_receiver(project): + source = dedent("""\ + class C: + def f(self, /, *, value): + local = value + 1 + return local + print(C().f(value=3)) + """) + resource = module(project, source) + project.do(LocalToField(project, resource, source.index("local") + 1).get_changes()) + assert "self.local" in resource.read() + assert execute(source) == execute(resource.read()) == "4\n" + + +@pytest.mark.parametrize( + "staticmethod, parameters", + [(False, "*, value"), (True, "value"), (True, "*, value")], +) +def test_local_to_field_refuses_missing_receiver(project, staticmethod, parameters): + decorator = "@staticmethod" if staticmethod else "" + source = dedent(f"""\ + class Value: + number = 3 + obj = Value() + class C: + {decorator} + def f({parameters}): + local = value.number + 1 + return local + print(C.f(value=obj)) + print(hasattr(obj, 'local')) + """) + resource = module(project, source) + assert execute(source) == "4\nFalse\n" + refused( + [resource], + lambda: LocalToField( + project, resource, source.index("local") + 1 + ).get_changes(), + ) + + +@pytest.mark.parametrize("parameters", ["cls, *, value", "cls, /, *, value"]) +def test_local_to_field_preserves_classmethod_receiver(project, parameters): + source = dedent(f"""\ + class Value: + number = 3 + obj = Value() + class C: + @classmethod + def f({parameters}): + local = value.number + 1 + return local + print(C.f(value=obj)) + print(hasattr(obj, 'local')) + print(hasattr(C, 'local')) + """) + resource = module(project, source) + project.do(LocalToField(project, resource, source.index("local") + 1).get_changes()) + assert "cls.local" in resource.read() + assert execute(source) == "4\nFalse\nFalse\n" + assert execute(resource.read()) == "4\nFalse\nTrue\n" diff --git a/ropetest/parameter_kinds_test.py b/ropetest/parameter_kinds_test.py new file mode 100644 index 000000000..93def3421 --- /dev/null +++ b/ropetest/parameter_kinds_test.py @@ -0,0 +1,1292 @@ +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base import arguments, builtins, evaluate, exceptions, pynames +from rope.base.project import Project +from rope.contrib.codeassist import code_assist, get_calltip, get_doc +from rope.refactor import change_signature, inline, rename + + +@pytest.fixture +def project(tmp_path): + project = Project( + str(tmp_path), save_objectdb=False, save_history=False, automatic_soa=False + ) + yield project + project.close() + + +def module(project, code, name="source"): + source = project.root.create_file(name + ".py") + source.write(dedent(code)) + return source + + +def execute(project, source): + result = subprocess.run( + [sys.executable, source.real_path], + cwd=project.address, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +@pytest.mark.parametrize("definition", ["def", "async def"]) +@pytest.mark.parametrize( + "parameters,call", [("target, /", "int"), ("*, target", "target=int")] +) +def test_outer_inline_preserves_parameter_binding( + project, definition, parameters, call +): + invocation = f"func({call})" + if definition == "async def": + invocation = f"asyncio.run({invocation})" + source = module( + project, + dedent(f"""\ + import asyncio + target = 42 + {definition} func({parameters}): + return target + print({invocation} is int) + print(target) + """), + ) + before = source.read() + assert execute(project, source) == "True\n42\n" + project.do( + inline.create_inline( + project, source, before.index("target =") + 1 + ).get_changes() + ) + assert source.read() != before + assert f"func({parameters})" in source.read() + assert "return target" in source.read() + assert execute(project, source) == "True\n42\n" + + +@pytest.mark.parametrize( + "parameters,call", [("target, /", "int"), ("*, target", "target=int")] +) +def test_parameter_identity_is_independent_of_outer_assignment( + project, parameters, call +): + source = module( + project, + dedent(f"""\ + target = 42 + def func({parameters}): + return target + print(func({call}) is int) + """), + ) + code = source.read() + pymodule = project.get_pymodule(source) + formal = evaluate.eval_location( + pymodule, code.index(parameters) + parameters.index("target") + 1 + ) + body = evaluate.eval_location( + pymodule, code.index("return target") + len("return ") + 1 + ) + assert isinstance(formal, pynames.ParameterName) + assert formal is body + assert formal is not pymodule["target"] + assert formal.get_definition_location() == (pymodule, 2) + assert execute(project, source) == "True\n" + with pytest.raises(exceptions.RefactoringError): + inline.InlineVariable( + project, source, code.index("return target") + len("return ") + 1 + ) + assert source.read() == code + assert execute(project, source) == "True\n" + + +def test_all_parameter_slots_keep_distinct_objects(project): + source = module( + project, + dedent("""\ + class Pos: + pass + class Normal: + pass + class Key: + pass + class Default: + pass + def func(pos, /, normal, *rest, key, default, **extras): + return pos + result = func(Pos(), Normal(), 42, default=Default(), key=Key(), extra="value") + print(isinstance(result, Pos)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Pos"].get_object() + function = pymodule["func"].get_object() + assert function.get_param_names(False) == ["pos", "normal", "key", "default"] + assert function.get_param_names() == [ + "pos", + "normal", + "key", + "default", + "rest", + "extras", + ] + parameters = function.get_parameters() + for index, (name, class_name) in enumerate( + [("pos", "Pos"), ("normal", "Normal"), ("key", "Key"), ("default", "Default")] + ): + assert parameters[name].index == index + assert ( + parameters[name].get_object().get_type() + is pymodule[class_name].get_object() + ) + assert parameters["rest"].index == 4 + assert isinstance(parameters["rest"].get_object().get_type(), builtins.List) + assert parameters["extras"].index == 5 + assert isinstance(parameters["extras"].get_object().get_type(), builtins.Dict) + + +def test_positional_only_keyword_does_not_replace_position_slot(project): + source = module( + project, + dedent("""\ + class Pos: + pass + class Extra: + pass + def func(target, /, **extras): + return target + result = func(Pos(), target=Extra()) + print(isinstance(result, Pos)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Pos"].get_object() + offset = source.read().index("target=Extra") + 1 + assert evaluate.eval_location(pymodule, offset) is None + + +def test_keyword_only_call_infers_actual_object(project): + source = module( + project, + dedent("""\ + class Value: + pass + def func(*, target): + return target + result = func(target=Value()) + print(isinstance(result, Value)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Value"].get_object() + + +@pytest.mark.parametrize("parameters", ["target=target, /", "*, target=target"]) +def test_default_reference_keeps_outer_identity_and_evaluation_time( + project, parameters +): + override = "func(target=int)" if parameters.startswith("*") else "func(int)" + source = module( + project, + dedent(f"""\ + events = [] + def make(): + events.append("created") + return 42 + target = make() + def func({parameters}): + return target + print(func(), func(), {override} is int, target, len(events)) + """), + ) + before = source.read() + assert execute(project, source) == "42 42 True 42 1\n" + pymodule = project.get_pymodule(source) + default_offset = before.index("target=target") + len("target=") + 1 + assert evaluate.eval_location(pymodule, default_offset) is pymodule["target"] + project.do( + rename.Rename(project, source, before.index("target =") + 1).get_changes( + "outer" + ) + ) + assert execute(project, source) == "42 42 True 42 1\n" + assert "target=outer" in source.read() + assert "return target" in source.read() + + +def test_parameter_rename_preserves_positional_only_kwargs_key(project): + source = module( + project, + dedent("""\ + target = 42 + def func(target, /, **extras): + return target, extras["target"] + result = func(int, target=str) + print(result[0] is int, result[1] is str, target) + """), + ) + before = source.read() + assert execute(project, source) == "True True 42\n" + project.do( + rename.Rename( + project, source, before.index("func(target") + len("func(") + 1 + ).get_changes("value") + ) + assert "target = 42" in source.read() + assert "target=str" in source.read() + assert 'extras["target"]' in source.read() + assert execute(project, source) == "True True 42\n" + + +def test_parameter_rename_updates_keyword_only_call(project): + source = module( + project, + dedent("""\ + target = 42 + def func(*, target): + return target + print(func(target=int) is int, target) + """), + ) + before = source.read() + assert execute(project, source) == "True 42\n" + project.do( + rename.Rename( + project, source, before.index("*, target") + len("*, ") + 1 + ).get_changes("value") + ) + assert "target = 42" in source.read() + assert "func(value=int)" in source.read() + assert execute(project, source) == "True 42\n" + + +def test_keyword_completion_uses_keyword_capable_parameters(project): + source = module( + project, + dedent("""\ + def func(pos, /, normal, *rest, key, **extras): + pass + func( + """), + ) + proposals = { + proposal.name + for proposal in code_assist( + project, source.read(), len(source.read()), resource=source + ) + } + assert {"normal=", "key="} <= proposals + assert not {"pos=", "rest=", "extras="} & proposals + + +@pytest.mark.parametrize("parameters,call", [("target=1, /", ""), ("*, target=1", "")]) +def test_unsupported_signature_rewrite_is_atomic(project, parameters, call): + source = module( + project, + dedent(f"""\ + def func({parameters}): + return target + print(func({call})) + """), + ) + before = source.read() + assert execute(project, source) == "1\n" + with pytest.raises(exceptions.RefactoringError): + changes = change_signature.ChangeSignature( + project, source, before.index("func") + 1 + ).get_changes([change_signature.ArgumentDefaultInliner(0)]) + project.do(changes) + assert source.read() == before + assert execute(project, source) == "1\n" + + +@pytest.mark.parametrize( + "decorator,parameters,call,receiver_name,receiver_is_class", + [ + ( + "", + "self, /, value, *, key", + "Child().func(Value(), key=Key())", + "self", + False, + ), + ( + "", + "self, /, value, *, key", + "Child.func(Child(), Value(), key=Key())", + "self", + False, + ), + ( + "@staticmethod", + "value, *, key", + "Child.func(Value(), key=Key())", + None, + False, + ), + ( + "@staticmethod", + "value, *, key", + "Child().func(Value(), key=Key())", + None, + False, + ), + ( + "@classmethod", + "cls, value, *, key", + "Child.func(Value(), key=Key())", + "cls", + True, + ), + ( + "@classmethod", + "cls, value, *, key", + "Child().func(Value(), key=Key())", + "cls", + True, + ), + ], +) +def test_method_call_slots_use_actual_receiver( + project, decorator, parameters, call, receiver_name, receiver_is_class +): + source = module( + project, + dedent(f"""\ + class Value: + pass + class Key: + pass + class Base: + {decorator} + def func({parameters}): + return value + class Child(Base): + pass + result = {call} + print(isinstance(result, Value)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Value"].get_object() + project.pycore.analyze_module(source) + function = pymodule["Base"].get_object()["func"].get_object() + bindings = function.get_parameters() + for name, expected in [("value", "Value"), ("key", "Key")]: + assert bindings[name].get_object().get_type() is pymodule[expected].get_object() + assert all( + value.get_type() is pymodule[expected].get_object() + for value in bindings[name].get_objects() + ) + if receiver_name is not None: + expected = pymodule["Child"].get_object() + receiver = bindings[receiver_name].get_object() + assert (receiver if receiver_is_class else receiver.get_type()) is expected + assert all( + (value if receiver_is_class else value.get_type()) is expected + for value in bindings[receiver_name].get_objects() + ) + + +def test_callable_attribute_adds_its_receiver_once(project): + source = module( + project, + dedent("""\ + class Value: + pass + class Key: + pass + class Callable: + def __call__(self, value, *, key): + return value + class Holder: + callback = Callable() + result = Holder().callback(Value(), key=Key()) + print(isinstance(result, Value)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Value"].get_object() + project.pycore.analyze_module(source) + function = pymodule["Callable"].get_object()["__call__"].get_object() + for name, expected in [("self", "Callable"), ("value", "Value"), ("key", "Key")]: + binding = function.get_parameters()[name] + assert binding.get_object().get_type() is pymodule[expected].get_object() + assert all( + value.get_type() is pymodule[expected].get_object() + for value in binding.get_objects() + ) + + +def test_defaults_follow_positional_tail_and_keyword_names(project): + source = module( + project, + dedent("""\ + class Pos: + pass + class Normal: + pass + class Key: + pass + def func(pos=Pos(), /, normal=Normal(), *rest, key=Key(), **extras): + return normal + result = func() + print(isinstance(result, Normal)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Normal"].get_object() + project.pycore.analyze_module(source) + for name, expected in [("pos", "Pos"), ("normal", "Normal"), ("key", "Key")]: + binding = pymodule["func"].get_object().get_parameters()[name] + assert binding.get_object().get_type() is pymodule[expected].get_object() + + +def test_keyword_completion_distinguishes_required_and_none_default(project): + source = module( + project, + dedent("""\ + def func(*, required, optional=None): + return required + print(func(required=int) is int) + """), + ) + assert execute(project, source) == "True\n" + code = source.read() + "func(" + proposals = { + proposal.name: proposal + for proposal in code_assist(project, code, len(code), resource=source) + } + assert proposals["required="].get_default() is None + assert proposals["optional="].get_default() == "None" + + +@pytest.mark.parametrize("unpacking", ["*supply()", "**supply()"]) +def test_unknown_unpacking_does_not_invent_default_bindings(project, unpacking): + source = module( + project, + dedent(f"""\ + class Pos: + pass + class Key: + pass + def supply(): + return eval("{'[]' if unpacking.startswith('*s') else '{}'}") + def func(pos=Pos(), /, *, key=Key()): + return pos + result = func({unpacking}) + print(isinstance(result, Pos)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + function = pymodule["func"].get_object() + call = pymodule.get_ast().body[-2].value + actual = arguments.create_arguments(None, function, call, pymodule.get_scope()) + objects = actual.get_arguments(function.get_param_names(False)) + if unpacking.startswith("*s"): + assert objects[0] is None + assert objects[1].get_type() is pymodule["Key"].get_object() + else: + assert objects[0].get_type() is pymodule["Pos"].get_object() + assert objects[1] is None + + +def test_variadic_receiver_does_not_occupy_keyword_only_slot(project): + source = module( + project, + dedent("""\ + class Key: + pass + class Owner: + def func(*items, key): + return key + result = Owner().func(key=Key()) + print(isinstance(result, Key)) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Key"].get_object() + project.pycore.analyze_module(source) + function = pymodule["Owner"].get_object()["func"].get_object() + assert ( + function.get_parameters()["key"].get_object().get_type() + is pymodule["Key"].get_object() + ) + assert isinstance( + function.get_parameters()["items"].get_object().get_type(), builtins.List + ) + + +@pytest.mark.parametrize( + "parameters,call", + [("target, /, **extras", "Value(), target=int"), ("*, target", "target=Value()")], +) +def test_dynamic_calls_record_ordinary_parameter_slots(project, parameters, call): + source = module( + project, + dedent(f"""\ + class Value: + pass + def func({parameters}): + return eval("target") + result = func({call}) + print(isinstance(result, Value)) + """), + ) + assert execute(project, source) == "True\n" + project.pycore.run_module(source).wait_process() + pymodule = project.get_pymodule(source) + function = pymodule["func"].get_object() + expected = pymodule["Value"].get_object() + assert function.get_parameters()["target"].get_object().get_type() is expected + assert function.get_parameters()["target"].get_objects() + assert all( + value.get_type() is expected + for value in function.get_parameters()["target"].get_objects() + ) + assert pymodule["result"].get_object().get_type() is expected + manager = project.pycore.object_info + path, key = manager._get_call_scope(function) + calls = list(manager.objectdb.get_callinfos(path, key)) + assert calls + for call_info in calls: + recorded = call_info.get_parameters() + assert len(recorded) == 1 + assert manager.to_pyobject(recorded[0]).get_type() is expected + + +def test_default_offsets_count_characters_after_non_ascii_prefix(project): + source = module( + project, + dedent("""\ + target = int + def func( + *, 中文="字", + value=("字", target)[1], + ): + return value + print(func() is int, func(value=str) is str) + """), + ) + before = source.read() + assert execute(project, source) == "True True\n" + pymodule = project.get_pymodule(source) + default_offset = before.index('"字", target') + len('"字", ') + 1 + assert evaluate.eval_location(pymodule, default_offset) is pymodule["target"] + project.do( + rename.Rename(project, source, before.index("target") + 1).get_changes("outer") + ) + assert execute(project, source) == "True True\n" + assert 'value=("字", outer)' in source.read() + + +@pytest.mark.parametrize("legacy", ["same_length", "short"]) +def test_changed_layout_isolates_legacy_calls_and_preserves_name_data(tmp_path, legacy): + folder = str(tmp_path) + project = Project( + folder, save_objectdb=True, save_history=False, automatic_soa=False + ) + source = module( + project, + dedent("""\ + class B: + pass + class Q: + pass + class K: + pass + def func(b, q, k): + return q + result = func(B(), Q(), K()) + print(isinstance(result, Q)) + """), + ) + if legacy == "short": + source.write( + source.read() + .replace("func(b, q, k)", "func(b)") + .replace("return q", "return b") + .replace("func(B(), Q(), K())", "func(B())") + .replace("result, Q", "result, B") + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + function = pymodule["func"].get_object() + previous_class = "Q" if legacy == "same_length" else "B" + assert ( + pymodule["result"].get_object().get_type() + is pymodule[previous_class].get_object() + ) + project.pycore.analyze_module(source) + manager = project.pycore.object_info + raw_scope = manager._get_scope(function) + assert manager._get_call_scope(function) == raw_scope + assert ( + manager.get_returned(function, None).get_type() + is pymodule[previous_class].get_object() + ) + manager.save_per_name( + function.get_scope(), "remembered", pymodule["B"].get_object() + ) + if legacy == "same_length": + replacement = ( + source.read() + .replace("func(b, q, k)", "func(b, *rest, k, q, **extras)") + .replace("return q", "return k") + .replace("func(B(), Q(), K())", "func(B(), q=Q(), k=K())") + .replace("result, Q", "result, K") + ) + else: + replacement = ( + source.read() + .replace("func(b)", "func(b, /, *, k)") + .replace("return b", "return k") + .replace("func(B())", "func(B(), k=K())") + .replace("result, B", "result, K") + ) + source.write(replacement) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + function = pymodule["func"].get_object() + assert manager.get_parameter_objects(function) is None + assert manager.get_returned(function, None) is None + assert ( + manager.get_per_name(function.get_scope(), "remembered") + is pymodule["B"].get_object() + ) + assert pymodule["result"].get_object().get_type() is pymodule["K"].get_object() + project.pycore.analyze_module(source) + manager.objectdb.validate_file(raw_scope[0]) + project.close() + reopened = Project( + folder, save_objectdb=True, save_history=False, automatic_soa=False + ) + try: + pymodule = reopened.get_pymodule(reopened.get_file("source.py")) + function = pymodule["func"].get_object() + manager = reopened.pycore.object_info + assert manager._get_call_scope(function) != manager._get_scope(function) + assert ( + manager.get_returned(function, None).get_type() + is pymodule["K"].get_object() + ) + assert ( + manager.get_per_name(function.get_scope(), "remembered") + is pymodule["B"].get_object() + ) + expected_parameters = [("b", "B"), ("k", "K")] + if legacy == "same_length": + expected_parameters.append(("q", "Q")) + for name, expected in expected_parameters: + assert ( + function.get_parameters()[name].get_object().get_type() + is pymodule[expected].get_object() + ) + finally: + reopened.close() + + +def test_nested_default_uses_enclosing_parameter_identity(project): + source = module( + project, + dedent("""\ + def outer(target, /): + def inner(*, target=target): + return target + return inner() + print(outer(int) is int, outer(str) is str) + """), + ) + before = source.read() + assert execute(project, source) == "True True\n" + pymodule = project.get_pymodule(source) + outer = pymodule["outer"].get_object() + default_offset = before.index("target=target") + len("target=") + 1 + assert ( + evaluate.eval_location(pymodule, default_offset) + is outer.get_parameters()["target"] + ) + project.do( + rename.Rename(project, source, before.index("target") + 1).get_changes("value") + ) + assert execute(project, source) == "True True\n" + assert "target=value" in source.read() + assert "return target" in source.read() + + +def test_positional_only_self_registers_instance_attributes(project): + source = module( + project, + dedent("""\ + class Value: + pass + class Owner: + def __init__(self, /, value): + self.value = value + result = Owner(Value()).value + print(isinstance(result, Value)) + """), + ) + assert execute(project, source) == "True\n" + project.pycore.analyze_module(source) + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object().get_type() is pymodule["Value"].get_object() + assert "value" in pymodule["Owner"].get_object() + + +def test_persisted_malformed_call_scopes_are_discarded(tmp_path): + folder = str(tmp_path) + project = Project( + folder, save_objectdb=True, save_history=False, automatic_soa=False + ) + source = module( + project, + dedent("""\ + class Value: + pass + def func(*, value): + return value + result = func(value=Value) + print(result is Value) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + assert pymodule["result"].get_object() is pymodule["Value"].get_object() + manager = project.pycore.object_info + function = pymodule["func"].get_object() + path, valid_key = manager._get_call_scope(function) + invalid_keys = [ + "!parameter-layout-v1:", + "!parameter-layout-v1:{broken", + "!parameter-layout-v1:[]", + '!parameter-layout-v1:["func","invalid",[]]', + '!parameter-layout-v1:["func","function",[["kwonly","previous"]]]', + ] + for key in invalid_keys: + manager.objectdb.add_pername( + path, key, "discarded", manager.to_textual(pymodule["Value"].get_object()) + ) + manager.save_per_name( + function.get_scope(), "remembered", pymodule["Value"].get_object() + ) + project.close() + reopened = Project( + folder, save_objectdb=True, save_history=False, automatic_soa=False + ) + try: + manager = reopened.pycore.object_info + assert set(invalid_keys) <= set(manager.objectdb.files[path]) + manager.objectdb.validate_file(path) + assert not set(invalid_keys) & set(manager.objectdb.files[path]) + assert valid_key in manager.objectdb.files[path] + pymodule = reopened.get_pymodule(reopened.get_file("source.py")) + function = pymodule["func"].get_object() + assert manager.get_returned(function, None) is pymodule["Value"].get_object() + assert ( + manager.get_per_name(function.get_scope(), "remembered") + is pymodule["Value"].get_object() + ) + finally: + reopened.close() + + +def test_ordinary_method_kinds_and_callable_use_correct_receivers(project): + source = module( + project, + dedent("""\ + class Value: + pass + class Base: + def normal(self, value): + return value + @staticmethod + def static(value): + return value + @classmethod + def class_method(cls, value): + return cls + class Child(Base): + pass + class Callable: + def __call__(self, value): + return value + class Holder: + callback = Callable() + normal_result = Child().normal(Value()) + static_result = Child().static(Value()) + class_result = Child.class_method(Value()) + callable_result = Holder().callback(Value()) + print(isinstance(normal_result, Value), isinstance(static_result, Value), class_result is Child, isinstance(callable_result, Value)) + """), + ) + assert execute(project, source) == "True True True True\n" + pymodule = project.get_pymodule(source) + for result in ["normal_result", "static_result", "callable_result"]: + assert ( + pymodule[result].get_object().get_type() is pymodule["Value"].get_object() + ) + assert pymodule["class_result"].get_object() is pymodule["Child"].get_object() + project.pycore.analyze_module(source) + manager = project.pycore.object_info + for class_name, method, receiver, receiver_type in [ + ("Base", "normal", "self", "Child"), + ("Base", "static", None, None), + ("Base", "class_method", "cls", "Child"), + ("Callable", "__call__", "self", "Callable"), + ]: + function = pymodule[class_name].get_object()[method].get_object() + assert manager._get_call_scope(function) != manager._get_scope(function) + for passed in function.get_parameters()["value"].get_objects(): + assert passed.get_type() is pymodule["Value"].get_object() + if receiver is not None: + for passed in function.get_parameters()[receiver].get_objects(): + actual = passed if receiver == "cls" else passed.get_type() + assert actual is pymodule[receiver_type].get_object() + + +@pytest.mark.parametrize("selected", ["formal", "body"]) +def test_defaulted_positional_parameter_rename_keeps_its_declaration(project, selected): + source = module( + project, + dedent("""\ + target = int + def func(target=target, /, **extras): + return target, extras["target"] + result = func(str, target=bytes) + print(result[0] is str, result[1] is bytes, target is int) + """), + ) + before = source.read() + assert execute(project, source) == "True True True\n" + pymodule = project.get_pymodule(source) + function = pymodule["func"].get_object() + parameter = function.get_parameters()["target"] + formal_offset = before.index("target=target") + for delta in [0, 1, 5]: + assert evaluate.eval_location(pymodule, formal_offset + delta) is parameter + body_offset = before.index("return target") + len("return ") + 1 + assert evaluate.eval_location(pymodule, body_offset) is parameter + assert ( + evaluate.eval_location(pymodule, formal_offset + len("target=") + 1) + is pymodule["target"] + ) + assert evaluate.eval_location(pymodule, before.index("target=bytes") + 1) is None + offset = formal_offset if selected == "formal" else body_offset + project.do(rename.Rename(project, source, offset).get_changes("local")) + assert execute(project, source) == "True True True\n" + assert "func(local=target, /, **extras)" in source.read() + assert "target=bytes" in source.read() + + +def test_default_and_same_line_body_call_keywords_keep_callee_identity(project): + source = module( + project, + dedent("""\ + def build(*, target): + return target + def func(target=build(target=int), /): return build(target=target) + print(func() is int, func(str) is str) + """), + ) + before = source.read() + assert execute(project, source) == "True True\n" + pymodule = project.get_pymodule(source) + parameter = pymodule["build"].get_object().get_parameters()["target"] + for keyword in ["target=int", "target=target)"]: + assert evaluate.eval_location(pymodule, before.index(keyword) + 1) is parameter + project.do( + rename.Rename( + project, source, before.index("*, target") + len("*, ") + 1 + ).get_changes("value") + ) + assert execute(project, source) == "True True\n" + assert ( + "func(target=build(value=int), /): return build(value=target)" in source.read() + ) + + +@pytest.mark.parametrize( + "parameters,display,without_self", + [ + ('self: "C", value: int = 1', "self, value=1", "value=1"), + ('self: "C", /', 'self: "C", /', ""), + ( + 'self: "C", /, value: int = 1', + 'self: "C", /, value: int = 1', + "value: int = 1", + ), + ( + 'self: "C", value: int = 1, /, *, key: str = "ok"', + 'self: "C", value: int = 1, /, *, key: str = "ok"', + 'value: int = 1, /, *, key: str = "ok"', + ), + ( + 'self: "C", *, value: int = 1', + 'self: "C", *, value: int = 1', + "*, value: int = 1", + ), + ], +) +def test_calltip_preserves_ordinary_display_and_parameter_separators( + project, parameters, display, without_self +): + result = "value" if "value" in parameters else "1" + source = module( + project, + dedent(f'''\ + class C: + def f({parameters}): + """Details.""" + return {result} + print(C().f()) + '''), + ) + code = source.read() + assert execute(project, source) == "1\n" + offset = code.rindex(".f(") + 1 + assert ( + get_calltip(project, code, offset, resource=source) == f"source.C.f({display})" + ) + assert ( + get_calltip(project, code, offset, resource=source, remove_self=True) + == f"source.C.f({without_self})" + ) + doc = get_doc(project, code, offset, resource=source) + assert doc.splitlines()[0] == f"C.f({display}):" + + +@pytest.mark.parametrize( + "parameters", + [ + 'self: "C"=((None)), /, *, payload=2', + 'self: ("C")=((lambda a, b: None)(None, None)), /, *, payload=2', + ], +) +def test_calltip_removes_complete_parenthesized_receiver(project, parameters): + source = module( + project, + dedent(f'''\ + class C: + def f({parameters}): + """Details.""" + return payload + print(C().f()) + '''), + ) + code = source.read() + assert execute(project, source) == "2\n" + offset = code.rindex(".f(") + 1 + assert ( + get_calltip(project, code, offset, resource=source) + == f"source.C.f({parameters})" + ) + assert ( + get_calltip(project, code, offset, resource=source, remove_self=True) + == "source.C.f(*, payload=2)" + ) + assert ( + get_doc(project, code, offset, resource=source).splitlines()[0] + == f"C.f({parameters}):" + ) + + +@pytest.mark.parametrize( + "declaration,decorator,parameters", + [ + ("", "staticmethod", "value, /"), + ("", "staticmethod", "value"), + ("static = staticmethod", "static", "value, /"), + ("from builtins import staticmethod as static", "static", "value, /"), + ("first = staticmethod; static = first", "static", "value, /"), + ("import builtins as namespace", "namespace.staticmethod", "value, /"), + ], +) +def test_static_parameter_attributes_do_not_belong_to_owner( + project, declaration, decorator, parameters +): + source = module( + project, + dedent(f"""\ + class Value: + pass + class Owner: + {declaration} + @{decorator} + def assign({parameters}): + value.field = 42 + pass + value = Value() + Owner.assign(value) + print(value.field) + try: + print(Owner.field) + except AttributeError: + print("absent") + """), + ) + before = source.read() + assert execute(project, source) == "42\nabsent\n" + owner = project.get_pymodule(source)["Owner"].get_object() + owner.get_scope().get_defined_names() + assert owner["assign"].get_object().get_kind() == "staticmethod" + assert "field" not in owner.get_attributes() + offset = before.index("print(Owner.field)") + len("print(Owner.") + 1 + with pytest.raises(exceptions.RefactoringError): + inline.create_inline(project, source, offset).get_changes() + assert source.read() == before + assert execute(project, source) == "42\nabsent\n" + + +@pytest.mark.parametrize( + "declaration,decorator,receiver,call,kind", + [ + ("", "", "self", "owner.assign()", "method"), + ("", "@classmethod", "cls", "Owner.assign()", "classmethod"), + ( + "class_method = classmethod", + "@class_method", + "cls", + "Owner.assign()", + "classmethod", + ), + ( + "import builtins as namespace", + "@namespace.classmethod", + "cls", + "Owner.assign()", + "classmethod", + ), + ( + "def staticmethod(func): return func", + "@staticmethod", + "self", + "owner.assign()", + "method", + ), + ], +) +def test_receiver_attributes_survive_kind_resolution_and_early_scope_access( + project, declaration, decorator, receiver, call, kind +): + inspected = "Owner" if receiver == "cls" else "owner" + source = module( + project, + dedent(f"""\ + class Owner: + {declaration} + {decorator} + def assign({receiver}, /): + {receiver}.field = 42 + owner = Owner() + {call} + print({inspected}.field) + """), + ) + assert execute(project, source) == "42\n" + owner = project.get_pymodule(source)["Owner"].get_object() + owner.get_scope().get_defined_names() + assert owner["assign"].get_object().get_kind() == kind + assert "field" in owner.get_attributes() + assert ( + owner["field"].get_object().get_type() is builtins.builtins["int"].get_object() + ) + assert execute(project, source) == "42\n" + + +def test_explicit_unknown_argument_does_not_use_default(project): + source = module( + project, + dedent("""\ + def supply(): + return eval("str") + def func(*, value=int): + return value + result = func(value=supply()) + print(result is str, func() is int) + """), + ) + assert execute(project, source) == "True True\n" + pymodule = project.get_pymodule(source) + function = pymodule["func"].get_object() + call = pymodule.get_ast().body[-2].value + actual = arguments.create_arguments(None, function, call, pymodule.get_scope()) + value = actual.get_arguments(function.get_param_names(False))[0] + assert value is None or value is not builtins.builtins["int"].get_object() + assert pymodule["result"].get_object() is not builtins.builtins["int"].get_object() + + +def test_variadic_method_with_no_ordinary_slots_keeps_receiver_in_items(project): + source = module( + project, + dedent("""\ + class Owner: + def func(*items): + return items[0] + owner = Owner() + result = owner.func() + print(result is owner) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + function = pymodule["Owner"].get_object()["func"].get_object() + actual = arguments.create_arguments( + pymodule["owner"], + function, + pymodule.get_ast().body[-2].value, + pymodule.get_scope(), + ) + assert function.get_param_names(False) == [] + assert actual.get_arguments(function.get_param_names(False)) == [] + assert isinstance( + function.get_parameters()["items"].get_object().get_type(), builtins.List + ) + assert execute(project, source) == "True\n" + + +@pytest.mark.parametrize("record_size", [1, 3]) +def test_persisted_wrong_size_calls_do_not_poison_readers_or_fallback( + tmp_path, record_size +): + folder = str(tmp_path) + project = Project( + folder, save_objectdb=True, save_history=False, automatic_soa=False + ) + source = module( + project, + dedent("""\ + class Pos: + pass + class Key: + pass + class Wrong: + pass + def func(pos, /, *, key): + return key + print(func(Pos, key=Key) is Key) + """), + ) + assert execute(project, source) == "True\n" + pymodule = project.get_pymodule(source) + function = pymodule["func"].get_object() + manager = project.pycore.object_info + path, key = manager._get_call_scope(function) + wrong = manager.to_textual(pymodule["Wrong"].get_object()) + manager.objectdb.add_callinfo(path, key, (wrong,) * record_size, wrong) + project.close() + reopened = Project( + folder, save_objectdb=True, save_history=False, automatic_soa=False + ) + try: + source = reopened.get_file("source.py") + pymodule = reopened.get_pymodule(source) + function = pymodule["func"].get_object() + manager = reopened.pycore.object_info + call = pymodule.get_ast().body[-1].value.args[0].left + actual = arguments.create_arguments(None, function, call, pymodule.get_scope()) + assert manager.get_parameter_objects(function) is None + assert manager.get_passed_objects(function, 0) == [] + assert manager.get_passed_objects(function, 1) == [] + assert manager.get_returned(function, None) is None + assert manager.get_exact_returned(function, actual) is None + pos, expected = pymodule["Pos"].get_object(), pymodule["Key"].get_object() + manager.function_called(function, [pos], expected) + assert manager.get_returned(function, None) is None + manager.function_called(function, [pos, expected], expected) + assert manager.get_parameter_objects(function) == [pos, expected] + assert manager.get_passed_objects(function, 1) == [expected] + assert manager.get_passed_objects(function, 2) == [] + assert manager.get_returned(function, None) is expected + assert manager.get_exact_returned(function, actual) is expected + assert execute(reopened, source) == "True\n" + finally: + reopened.close() + + +def test_calltip_without_positional_receiver_keeps_keyword_only_parameter(project): + source = module( + project, + dedent("""\ + class Owner: + def func(*, value: int = 2): + return value + print(Owner.func(value=2)) + """), + ) + code = source.read() + assert execute(project, source) == "2\n" + offset = code.rindex(".func(") + 1 + assert get_calltip(project, code, offset, resource=source, remove_self=True) == ( + "source.Owner.func(*, value: int = 2)" + ) + + +def test_direct_inline_parameter_on_assignment_preserves_runtime(project): + source = module(project, "target = int\nprint(target is int)\n") + before = source.read() + assert execute(project, source) == "True\n" + with pytest.raises(exceptions.RefactoringError): + inline.InlineParameter(project, source, before.index("target") + 1) + assert source.read() == before + assert execute(project, source) == "True\n" + + +def test_keyword_only_without_varargs_parameter_info(project): + from rope.base import libutils + + mod = libutils.get_string_module( + project, + "def func(first, /, *, named):\n return first, named\n", + ) + function = mod["func"].get_object() + assert function.get_param_names() == ["first", "named"] + assert set(function.get_parameters()) == {"first", "named"} + + +def test_keyword_only_method_without_positional_receiver(project): + from rope.base import libutils + + mod = libutils.get_string_module( + project, + "class Sample:\n" + " def method(*, named: str):\n" + " return named\n", + ) + method = mod["Sample"].get_object()["method"].get_object() + assert method.get_param_names() == ["named"] + assert ( + method.get_parameters()["named"].get_object().get_type() + != mod["Sample"].get_object() + ) + + +def test_keyword_only_call_information_skips_varargs_slot(project): + code = dedent("""\ + class C: + pass + def func(first, /, *items, named=None): + return named + func(1, 2, named=C()) + """) + source = module(project, code) + project.pycore.analyze_module(source) + pymod = project.get_pymodule(source) + c_class = pymod["C"].get_object() + scope = pymod["func"].get_object().get_scope() + assert scope["named"].get_object().get_type() == c_class diff --git a/ropetest/projecttest.py b/ropetest/projecttest.py index 1f1b37d83..8e84a4f31 100644 --- a/ropetest/projecttest.py +++ b/ropetest/projecttest.py @@ -2,14 +2,15 @@ import pathlib import tempfile import unittest +import warnings from textwrap import dedent from unittest.mock import patch from rope.base.exceptions import ResourceNotFoundError, RopeError from rope.base.fscommands import FileSystemCommands -from rope.base.libutils import path_to_resource +from rope.base.libutils import path_relative_to_project_root, path_to_resource, relative from rope.base.project import NoProject, Project, _realpath -from rope.base.resourceobserver import FilteredResourceObserver +from rope.base.resourceobserver import ChangeIndicator, FilteredResourceObserver from rope.base.resources import File, Folder from ropetest import testutils @@ -115,9 +116,8 @@ def test_creating_file_inside_folder(self): file = self.project.get_resource(file_path) file.write("sample notes") self.assertEqual(file_path, file.path) - self.assertEqual( - "sample notes", open(os.path.join(self.project.address, file_path)).read() - ) + with open(os.path.join(self.project.address, file_path)) as written: + self.assertEqual("sample notes", written.read()) def test_failing_when_creating_file_inside_non_existent_folder(self): with self.assertRaises(ResourceNotFoundError): @@ -555,6 +555,45 @@ def test_simple_path_to_resource(self): myfolder, path_to_resource(self.project, myfolder.real_path, type="folder") ) + def test_project_relative_paths_do_not_warn(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.assertEqual( + "", path_relative_to_project_root(self.project, self.project.address) + ) + self.assertEqual( + "sample_folder/child.py", + path_relative_to_project_root( + self.project, + os.path.join(self.project.address, "sample_folder", "child.py"), + ), + ) + self.assertEqual( + self.project.get_resource(self.sample_file), + path_to_resource(self.project, self.sample_path), + ) + self.assertEqual([], caught) + + def test_project_relative_paths_reject_outside_paths(self): + paths = [ + os.path.dirname(self.project.address), + self.project.address + "-sibling/child.py", + os.path.join(self.project.address, os.pardir, "outside.py"), + ] + if os.name == "nt": + drive = os.path.splitdrive(self.project.address)[0] + other_drive = "D:" if drive.upper() == "C:" else "C:" + paths.append(other_drive + "\\outside.py") + for path in paths: + with self.subTest(path=path): + self.assertIsNone(path_relative_to_project_root(self.project, path)) + + def test_legacy_relative_path_warns(self): + with self.assertWarnsRegex(DeprecationWarning, "relative is deprecated"): + self.assertEqual( + self.sample_file, relative(self.project.address, self.sample_path) + ) + @testutils.skipNotPOSIX() def test_ignoring_symlinks_inside_project(self): project2 = testutils.sample_project(folder_name="sampleproject2") @@ -962,7 +1001,7 @@ def test_out_of_project_files_and_path_to_resource(self): ) -class _MockFSCommands: +class _MockFSCommands(FileSystemCommands): def __init__(self): self.log = "" self.fscommands = FileSystemCommands() @@ -988,7 +1027,7 @@ def read(self, path): return self.fscommands.read(path) -class _DeprecatedFSCommands: +class _WriteTracingFSCommands(FileSystemCommands): def __init__(self): self.log = "" self.fscommands = FileSystemCommands() @@ -1036,6 +1075,21 @@ def test_setting_ignored_resources(self): self.assertTrue(self.project.is_ignored(myfile)) self.assertFalse(self.project.is_ignored(file2)) + def test_python_path_under_root_without_ignored_resources_pref(self): + # A python path folder inside the root whose path contains + # site-packages must not require the caller to preseed the + # ignored_resources preference (#756 follow-up). + import tempfile + from pathlib import Path + + directory = Path(tempfile.mkdtemp()) + site_packages = directory / ".venv" / "lib" / "python3.13" / "site-packages" + site_packages.mkdir(parents=True) + (site_packages / "stubpkg.py").write_text("") + project = Project(str(directory), python_path=[str(site_packages)]) + self.assertTrue(project.get_file(".venv/lib/python3.13/site-packages/stubpkg.py").exists() is not None) + project.close() + def test_ignored_folders(self): self.project = testutils.sample_project(ignored_resources=["myfolder"]) myfolder = self.project.root.create_folder("myfolder") @@ -1077,25 +1131,112 @@ def test_star_should_not_include_slashes(self): def test_normal_fscommands(self): fscommands = _MockFSCommands() self.project = testutils.sample_project(fscommands=fscommands) + setup_log = fscommands.log + self.assertIn("create_folder ", setup_log) + self.assertTrue(self.project.root.exists()) myfile = self.project.get_file("myfile.txt") myfile.create() - self.assertEqual("create_file ", fscommands.log) + self.assertEqual(setup_log + "create_file ", fscommands.log) + self.assertTrue(myfile.exists()) def test_fscommands_and_ignored_resources(self): fscommands = _MockFSCommands() self.project = testutils.sample_project( fscommands=fscommands, ignored_resources=["myfile.txt"], ropefolder=None ) + setup_log = fscommands.log + self.assertIn("create_folder ", setup_log) + self.assertTrue(self.project.root.exists()) myfile = self.project.get_file("myfile.txt") myfile.create() - self.assertEqual("", fscommands.log) + self.assertEqual(setup_log, fscommands.log) + self.assertTrue(myfile.exists()) - def test_deprecated_fscommands(self): - fscommands = _DeprecatedFSCommands() + def test_write_tracing_fscommands(self): + fscommands = _WriteTracingFSCommands() self.project = testutils.sample_project(fscommands=fscommands) + setup_log = fscommands.log + self.assertIn("create_folder ", setup_log) + self.assertTrue(self.project.root.exists()) + myfile = self.project.get_file("myfile.txt") + myfile.create() + self.assertEqual(setup_log + "create_file ", fscommands.log) + self.assertTrue(myfile.exists()) + + def test_get_resource_queries_the_fscommands_owner(self): + class _RecordingFSCommands(FileSystemCommands): + def __init__(self): + self.queries = [] + + def exists(self, path): + self.queries.append(("exists", path)) + return FileSystemCommands.exists(path) + + def isfile(self, path): + self.queries.append(("isfile", path)) + return FileSystemCommands.isfile(path) + + def isdir(self, path): + self.queries.append(("isdir", path)) + return FileSystemCommands.isdir(path) + + fscommands = _RecordingFSCommands() + self.project = testutils.sample_project(fscommands=fscommands, ropefolder=None) + myfile = self.project.get_file("myfile.txt") + myfile.create() + self.project.root.create_folder("myfolder") + + file_resource = self.project.get_resource("myfile.txt") + folder_resource = self.project.get_resource("myfolder") + + file_path = self.project._get_resource_path("myfile.txt") + folder_path = self.project._get_resource_path("myfolder") + self.assertEqual(file_resource, myfile) + self.assertIn(("exists", file_path), fscommands.queries) + self.assertIn(("isfile", file_path), fscommands.queries) + self.assertIn(("exists", folder_path), fscommands.queries) + self.assertIn(("isdir", folder_path), fscommands.queries) + self.assertTrue(folder_resource.is_folder()) + + def test_folder_children_are_enumerated_by_the_fscommands_owner(self): + class _RecordingFSCommands(FileSystemCommands): + def __init__(self): + self.queries = [] + + def listdir(self, path): + self.queries.append(("listdir", path)) + return FileSystemCommands.listdir(path) + + fscommands = _RecordingFSCommands() + self.project = testutils.sample_project(fscommands=fscommands, ropefolder=None) myfile = self.project.get_file("myfile.txt") myfile.create() - self.assertEqual("create_file ", fscommands.log) + myfolder = self.project.root.create_folder("myfolder") + + children = {resource.path for resource in self.project.root.get_children()} + + self.assertEqual({"myfile.txt", "myfolder"}, children) + self.assertIn(("listdir", self.project.root.real_path), fscommands.queries) + + def test_change_indicator_reads_metadata_from_the_fscommands_owner(self): + class _RecordingFSCommands(FileSystemCommands): + def __init__(self): + self.stats = [] + + def stat(self, path): + self.stats.append(path) + return FileSystemCommands.stat(path) + + fscommands = _RecordingFSCommands() + self.project = testutils.sample_project(fscommands=fscommands, ropefolder=None) + myfile = self.project.get_file("myfile.txt") + myfile.create() + myfile.write("contents\n") + + indicator = ChangeIndicator() + indicator.get_indicator(myfile) + + self.assertEqual([myfile.real_path], fscommands.stats) def test_ignored_resources_and_prefixes(self): self.project = testutils.sample_project(ignored_resources=[".hg"]) @@ -1172,6 +1313,29 @@ def test_loading_pyproject_no_tool_rope_section(self): myfile = self.project.get_file("pyproject.py") self.assertFalse(self.project.is_ignored(myfile)) + def test_explicit_prefs_without_config_files_ignore_an_invalid_pyproject(self): + self.project = testutils.sample_project() + config = self.project.get_file("pyproject.toml") + if not config.exists(): + config.create() + config.write(dedent("""\ + [tool.rope] + ignored_resources=["pyproject.py"] + <<<<<<< HEAD + ======= + >>>>>>> incoming + """)) + self.project.close() + self.project = Project( + self.project.address, + config_files=False, + ignored_resources=["explicit.py"], + ) + self.assertTrue(self.project.is_ignored(self.project.get_file("explicit.py"))) + self.assertFalse( + self.project.is_ignored(self.project.get_file("pyproject.py")) + ) + def test_ignoring_syntax_errors(self): self.project = testutils.sample_project( ropefolder=None, ignore_syntax_errors=True diff --git a/ropetest/public_resource_contracttest.py b/ropetest/public_resource_contracttest.py new file mode 100644 index 000000000..5cabbe989 --- /dev/null +++ b/ropetest/public_resource_contracttest.py @@ -0,0 +1,40 @@ +"""Public resource contracts used by project consumers.""" + +import pytest + +from rope.base.exceptions import ResourceNotFoundError +from rope.base.project import Project +from rope.base.resources import File, Folder + + +def test_public_package_and_module_resources(tmp_path): + package = tmp_path / "package" + package.mkdir() + (package / "__init__.py").write_text("") + (tmp_path / "module.py").write_text("") + initializer_directory = tmp_path / "initializer_directory" + initializer_directory.mkdir() + (initializer_directory / "__init__.py").mkdir() + missing_initializer = tmp_path / "missing_initializer" + missing_initializer.mkdir() + + project = Project(tmp_path, ropefolder=None, source_folders=[""]) + try: + package_resource = project.find_module("package") + module_resource = project.find_module("module") + assert isinstance(package_resource, Folder) + assert isinstance(module_resource, File) + assert package_resource.has_child("__init__.py") + assert not package_resource.get_child("__init__.py").is_folder() + assert not module_resource.is_folder() + assert project.get_folder("initializer_directory").get_child( + "__init__.py" + ).is_folder() + assert not project.get_folder("missing_initializer").has_child("__init__.py") + assert project.find_module("absent_module") is None + with pytest.raises(ResourceNotFoundError): + package_resource.get_child("absent_child") + finally: + project.close() + assert project.ropefolder is None + assert not (tmp_path / ".ropeproject").exists() diff --git a/ropetest/pycoretest.py b/ropetest/pycoretest.py index 5ccd80d2a..c7a2d1a2c 100644 --- a/ropetest/pycoretest.py +++ b/ropetest/pycoretest.py @@ -1,16 +1,57 @@ +import collections.abc +import os import sys +import tempfile +import typing import unittest from textwrap import dedent -from rope.base import exceptions, libutils -from rope.base.builtins import BuiltinClass, File +from rope.base import evaluate, exceptions, libutils, pynames +from rope.base.builtins import BuiltinClass, File, builtins +from rope.base.project import Project from rope.base.pycore import _TextChangeDetector from rope.base.pynamesdef import AssignedName -from rope.base.pyobjects import AbstractFunction, get_base_type +from rope.base.pyobjects import ( + AbstractFunction, + UnresolvedBase, + get_base_type, + get_unknown, +) from ropetest import testutils class PyCoreTest(unittest.TestCase): + def test_stdlib_runtime_module_election_preserves_declaring_source(self): + runtime = collections.abc + self.assertIs(runtime, sys.modules[runtime.__spec__.name]) + result = self.project.get_module('collections.abc') + # Same file, whatever spelling the runtime used (Windows: lib vs Lib). + self.assertTrue(os.path.samefile(runtime.__file__, result.get_resource().real_path)) + self.assertIn('Sequence', result) + + def test_stdlib_module_election_does_not_cross_project_parent_shadow(self): + package = testutils.create_package(self.project, 'collections') + package.get_child('__init__.py').write("raise AssertionError('target code must not execute')\n") + with self.assertRaises(exceptions.ModuleNotFoundError): + self.project.get_module('collections.abc') + + def test_stdlib_module_election_does_not_cross_origin_shadow(self): + import collections.abc + + import _collections_abc + + resource = testutils.create_module(self.project, '_collections_abc') + resource.write("raise AssertionError('target code must not execute')\n") + if collections.abc is _collections_abc: + # Python 3.13+ aliases collections.abc to _collections_abc, so a + # project module of that name would cross the stdlib origin. + with self.assertRaises(exceptions.ModuleNotFoundError): + self.project.get_module('collections.abc') + else: + # Earlier versions ship collections/abc.py as its own module. + module = self.project.get_module('collections.abc') + self.assertNotEqual(resource, module.get_resource()) + def setUp(self): super().setUp() self.project = testutils.sample_project() @@ -255,6 +296,1285 @@ def f(self): derived = mod["Derived"].get_object() self.assertTrue("f" in derived) + def _assert_mro_matches_python(self, code, names): + namespace = {} + exec(code, namespace) + mod = libutils.get_string_module(self.project, code) + for name in names: + with self.subTest(name=name): + expected = [cls.__name__ for cls in namespace[name].__mro__] + actual = [cls.get_name() for cls in mod[name].get_object().get_mro()] + self.assertEqual(expected, actual) + + def test_bases_through_annotated_type_aliases_resolve_the_class(self): + code = dedent("""\ + import typing + from typing import TypeAlias as Declared + class Parts: + class Root[T]: + pass + class Facade(Parts): + Root: Declared = Parts.Root + Alias: typing.TypeAlias = Parts.Root + class ViaFacade(Facade.Root[int]): + pass + class ViaModule(Alias[int]): + pass + class Plain(Alias): + pass + """) + mod = libutils.get_string_module(self.project, code) + root = mod["Parts"].get_object()["Root"].get_object() + for name in ("ViaFacade", "ViaModule", "Plain"): + with self.subTest(name=name): + self.assertEqual([root], mod[name].get_object().get_superclasses()) + + def test_type_checking_branch_is_taken_by_static_inference(self): + # PEP 484: static analysis assumes typing.TYPE_CHECKING is true. + guards = ( + ("from typing import TYPE_CHECKING", "TYPE_CHECKING"), + ("from typing import TYPE_CHECKING as TC", "TC"), + ("import typing", "typing.TYPE_CHECKING"), + ("import typing as t", "t.TYPE_CHECKING"), + ) + for guard_import, guard in guards: + with self.subTest(guard=guard): + code = dedent(f"""\ + {guard_import} + from typing import TypeAlias + class Parts: + class Root[T]: + pass + class Facade(Parts): + if {guard}: + Root: TypeAlias = Parts.Root + Plain = Parts.Root + if {guard}: + Alias: TypeAlias = Parts.Root + else: + Alias = None + class ViaFacade(Facade.Root): + pass + class ViaGeneric(Facade.Root[int]): + pass + class ViaModule(Alias): + pass + """) + mod = libutils.get_string_module(self.project, code) + root = mod["Parts"].get_object()["Root"].get_object() + facade = mod["Facade"].get_object() + self.assertIs(root, facade["Root"].get_object()) + self.assertIs(root, facade["Plain"].get_object()) + self.assertIs(root, mod["Alias"].get_object()) + for name in ("ViaFacade", "ViaGeneric", "ViaModule"): + derived = mod[name].get_object() + self.assertEqual([root], derived.get_superclasses()) + self.assertEqual([derived, *root.get_mro()], derived.get_mro()) + + def test_name_spelled_type_checking_outside_typing_keeps_both_arms(self): + code = dedent("""\ + TYPE_CHECKING = False + class Root: + pass + if TYPE_CHECKING: + Alias = Root + """) + mod = libutils.get_string_module(self.project, code) + self.assertIs(get_unknown(), mod["Alias"].get_object()) + + def test_type_checking_body_wins_over_the_else_arm(self): + code = dedent("""\ + from typing import TYPE_CHECKING + class Checked: + pass + class Runtime: + pass + if TYPE_CHECKING: + Alias = Checked + else: + Alias = Runtime + """) + mod = libutils.get_string_module(self.project, code) + self.assertIs(mod["Checked"].get_object(), mod["Alias"].get_object()) + + def test_branch_test_reading_the_inferred_name_keeps_agreeing_arms(self): + code = dedent("""\ + class A: + flag = True + a = A + if a.flag: + a = A + """) + mod = libutils.get_string_module(self.project, code) + self.assertIs(mod["A"].get_object(), mod["a"].get_object()) + + def test_project_typing_module_without_type_checking(self): + testutils.create_module(self.project, "typing").write("x = 1\n") + code = dedent("""\ + import os + from typing import TYPE_CHECKING + flag = os.environ.get("FLAG") + class A: + pass + if flag: + X = A + if TYPE_CHECKING: + Y = A + """) + mod = libutils.get_string_module(self.project, code) + self.assertIs(get_unknown(), mod["X"].get_object()) + self.assertIs(get_unknown(), mod["Y"].get_object()) + + def test_type_checking_rebound_by_typing_extensions_is_taken(self): + testutils.create_module(self.project, "typing_extensions").write( + dedent("""\ + import typing + TYPE_CHECKING = typing.TYPE_CHECKING + """) + ) + code = dedent("""\ + from typing_extensions import TYPE_CHECKING + class Root: + pass + if TYPE_CHECKING: + Alias = Root + """) + mod = libutils.get_string_module(self.project, code) + self.assertIs(mod["Root"].get_object(), mod["Alias"].get_object()) + + def test_type_checking_name_bound_more_than_once_stays_undecided(self): + rebinds = ( + "TC &= False", + "def rebind():\n global TC\n TC = False", + "del TC", + ) + for rebind in rebinds: + with self.subTest(rebind=rebind): + code = "\n".join(( + "import typing", + "TC = typing.TYPE_CHECKING", + rebind, + "class Checked:", + " pass", + "class Runtime:", + " pass", + "if TC:", + " Alias = Checked", + "else:", + " Alias = Runtime", + "", + )) + mod = libutils.get_string_module(self.project, code) + self.assertIs(get_unknown(), mod["Alias"].get_object()) + + def test_type_checking_name_shadowed_in_another_scope_is_taken(self): + code = dedent("""\ + import typing + TC = typing.TYPE_CHECKING + def shadow(): + TC = False + class Root: + TC = False + if TC: + Alias = Root + """) + mod = libutils.get_string_module(self.project, code) + self.assertIs(mod["Root"].get_object(), mod["Alias"].get_object()) + + def test_type_checking_decision_follows_the_module_that_binds_it(self): + guard = testutils.create_module(self.project, "guard") + guard.write("import typing\nTYPE_CHECKING = typing.TYPE_CHECKING\n") + user = testutils.create_module(self.project, "user") + user.write( + dedent("""\ + from guard import TYPE_CHECKING + class Checked: + pass + class Runtime: + pass + if TYPE_CHECKING: + Alias = Checked + else: + Alias = Runtime + """) + ) + mod = self.project.get_pymodule(user) + self.assertIs(mod["Checked"].get_object(), mod["Alias"].get_object()) + guard.write("TYPE_CHECKING = False\n") + mod = self.project.get_pymodule(user) + self.assertIs(get_unknown(), mod["Alias"].get_object()) + guard.write("import typing\nTYPE_CHECKING = typing.TYPE_CHECKING\n") + mod = self.project.get_pymodule(user) + self.assertIs(mod["Checked"].get_object(), mod["Alias"].get_object()) + + def test_type_checking_decision_follows_a_project_typing_module(self): + shadow = testutils.create_module(self.project, "typing") + shadow.write("x = 1\n") + user = testutils.create_module(self.project, "user") + user.write( + dedent("""\ + from typing import TYPE_CHECKING + class Checked: + pass + class Runtime: + pass + if TYPE_CHECKING: + Alias = Checked + else: + Alias = Runtime + """) + ) + mod = self.project.get_pymodule(user) + self.assertIs(get_unknown(), mod["Alias"].get_object()) + shadow.remove() + mod = self.project.get_pymodule(user) + self.assertIs(mod["Checked"].get_object(), mod["Alias"].get_object()) + + def test_annotation_named_like_type_alias_still_types_an_instance(self): + code = dedent("""\ + class TypeAlias: + pass + class Root: + pass + value: TypeAlias = Root + """) + mod = libutils.get_string_module(self.project, code) + self.assertEqual( + mod["TypeAlias"].get_object(), mod["value"].get_object().get_type() + ) + + def test_base_type_markers_are_not_class_objects(self): + for name in ("Type", "Module", "Function", "Unknown"): + with self.subTest(marker=name): + marker = get_base_type(name) + self.assertEqual(get_base_type("Type"), marker.get_type()) + mod = libutils.get_string_module( + self.project, "class Derived(marker):\n pass\n" + ) + mod.get_scope().get_names()["marker"] = pynames.UnboundName(marker) + derived = mod["Derived"].get_object() + (unresolved,) = derived.get_bases() + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertIs(marker, unresolved.pyobject) + self.assertEqual([], derived.get_superclasses()) + self.assertEqual([derived, unresolved], derived.get_mro()) + + def test_imported_nonclass_bases_preserve_resolved_ancestors(self): + provider = testutils.create_module(self.project, "provider") + provider.write("def function():\n pass\n") + for expression in ("provider", "provider.function", "provider.missing"): + with self.subTest(expression=expression): + code = dedent(f"""\ + import provider + class Base(object): + pass + class Derived(Base, {expression}): + pass + """) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + derived = mod["Derived"].get_object() + resolved, unresolved = derived.get_bases() + self.assertIs(base, resolved) + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertEqual([base], derived.get_superclasses()) + self.assertEqual( + [derived, *base.get_mro(), unresolved], + derived.get_mro(), + ) + + def test_mro_of_diamond_inheritance(self): + code = dedent("""\ + class A(object): + pass + class B(A): + pass + class C(A): + pass + class D(B, C): + pass + """) + self._assert_mro_matches_python(code, ["A", "B", "C", "D"]) + + def test_mro_matches_python_for_nested_hierarchies(self): + code = dedent("""\ + class O(object): + pass + class A(O): + pass + class B(O): + pass + class C(O): + pass + class D(O): + pass + class E(O): + pass + class K1(A, B, C): + pass + class K2(D, B, E): + pass + class K3(D, A): + pass + class Z(K1, K2, K3): + pass + """) + self._assert_mro_matches_python(code, ["K1", "K2", "K3", "Z"]) + + def test_mro_matches_python_for_builtin_bases(self): + code = dedent("""\ + class Base(dict): + pass + class Mixin(object): + pass + class Derived(Mixin, Base): + pass + class Flag(Mixin, int): + pass + class Failure(Mixin, KeyError): + pass + """) + self._assert_mro_matches_python(code, ["Base", "Derived", "Flag", "Failure"]) + + def test_builtin_ancestors_contribute_only_their_own_attributes(self): + code = dedent("""\ + class Base(object): + def __init__(self): + pass + def __init_subclass__(cls): + pass + class Mixin(object): + pass + class Derived(Mixin, Base): + pass + class Mapping(dict, Base): + pass + """) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + derived = mod["Derived"].get_object() + mapping = mod["Mapping"].get_object() + self.assertEqual(base["__init__"].get_object(), derived["__init__"].get_object()) + self.assertEqual( + base["__init_subclass__"].get_object(), + mapping["__init_subclass__"].get_object(), + ) + self.assertNotEqual( + base["__init__"].get_object(), mapping["__init__"].get_object() + ) + + def test_subscripted_class_base_calls_implicit_class_getitem(self): + code = dedent("""\ + class Base: + def __class_getitem__(cls, item): + return cls + class Derived(Base[int]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Base"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + derived = mod["Derived"].get_object() + self.assertEqual([derived, base], derived.get_mro()) + + def test_type_of_class_uses_its_metaclass_without_changing_kind(self): + code = dedent("""\ + class Meta(type): + pass + class Base(metaclass=Meta): + pass + selected = type(Base) + instance_type = type(Base()) + meta_type = type(Meta) + created = Meta("Created", (), {}) + created_type = type(created) + dynamic = type("Dynamic", (), {}) + dynamic_type = type(dynamic) + """) + namespace = {} + exec(code, namespace) + self.assertIs(namespace["selected"], namespace["Meta"]) + self.assertIs(namespace["instance_type"], namespace["Base"]) + self.assertIs(namespace["meta_type"], type) + self.assertIs(namespace["created_type"], namespace["Meta"]) + self.assertIs(namespace["dynamic_type"], type) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + meta = mod["Meta"].get_object() + self.assertIs(mod["selected"].get_object(), meta) + self.assertIs(mod["instance_type"].get_object(), base) + self.assertIs(mod["meta_type"].get_object(), builtins["type"].get_object()) + self.assertIs(mod["created_type"].get_object(), meta) + self.assertIs(mod["dynamic_type"].get_object(), builtins["type"].get_object()) + self.assertIs(base.get_type(), get_base_type("Type")) + self.assertIs(meta.get_type(), get_base_type("Type")) + + def test_type_of_class_preserves_unresolved_metaclass_witness(self): + code = dedent("""\ + from builtins import missing + class Base(missing): + pass + selected = type(Base) + """) + with self.assertRaises(ImportError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + (unresolved,) = base.get_bases() + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertIs(mod["selected"].get_object(), unresolved) + + def test_type_of_builtin_instances_returns_the_canonical_class(self): + code = dedent("""\ + number_type = type(1) + text_type = type("value") + list_type = type([1]) + dict_type = type({"key": 1}) + tuple_type = type((1, 2)) + """) + namespace = {} + exec(code, namespace) + mod = libutils.get_string_module(self.project, code) + for name, expected in ( + ("number_type", int), + ("text_type", str), + ("list_type", list), + ("dict_type", dict), + ("tuple_type", tuple), + ): + with self.subTest(name=name): + self.assertIs(namespace[name], expected) + self.assertIs( + mod[name].get_object(), builtins[expected.__name__].get_object() + ) + + def test_subscripted_class_base_uses_class_getitem_result(self): + code = dedent("""\ + class Other: + pass + class Base: + def __class_getitem__(cls, item): + return Other + class Derived(Base[int]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + other = mod["Other"].get_object() + derived = mod["Derived"].get_object() + self.assertEqual([derived, other], derived.get_mro()) + + def test_subscripted_class_base_prefers_metaclass_getitem(self): + code = dedent("""\ + class Other: + pass + class Meta(type): + def __getitem__(cls, item): + return Other + class Base(metaclass=Meta): + def __class_getitem__(cls, item): + return cls + class Derived(Base[int]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + other = mod["Other"].get_object() + derived = mod["Derived"].get_object() + self.assertEqual([derived, other], derived.get_mro()) + + def test_subscripted_class_metaclass_classmethod_receives_metaclass(self): + code = dedent("""\ + class Meta(type): + @classmethod + def __getitem__(cls, item): + return cls + class Base(metaclass=Meta): + pass + class Derived(Base[int]): + pass + """) + self._assert_mro_matches_python(code, ["Derived"]) + + def test_subscripted_class_retains_unresolved_metaclass_base(self): + code = dedent("""\ + from builtins import missing + class Base(missing): + def __class_getitem__(cls, item): + return cls + selected = Base[int] + """) + with self.assertRaises(ImportError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + (unresolved,) = base.get_bases() + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertIs(base.get_metaclass(), unresolved) + self.assertIs(mod["selected"].get_object(), unresolved) + + def test_subscripted_class_does_not_guess_an_unknown_explicit_metaclass(self): + code = dedent("""\ + from builtins import missing + class Base(metaclass=missing): + def __class_getitem__(cls, item): + return cls + selected = Base[int] + """) + with self.assertRaises(ImportError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + unresolved = mod["Base"].get_object().get_metaclass() + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertEqual("missing", unresolved.node.id) + self.assertIs(mod["selected"].get_object(), unresolved) + + def test_subscripted_class_static_hook_receives_only_the_key(self): + code = dedent("""\ + class Other: + pass + class Base: + @staticmethod + def __class_getitem__(item): + return item + class Derived(Base[Other]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + other = mod["Other"].get_object() + derived = mod["Derived"].get_object() + self.assertEqual([derived, other], derived.get_mro()) + + def test_subscripted_class_static_lambda_receives_only_the_key(self): + code = dedent("""\ + class Other: + pass + class Base: + __class_getitem__ = staticmethod(lambda item: item) + class Derived(Base[Other]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual([derived, mod["Other"].get_object()], derived.get_mro()) + + def test_subscripted_metaclass_class_lambda_receives_its_metaclass(self): + code = dedent("""\ + class Meta(type): + __getitem__ = classmethod(lambda cls, item: cls) + class Base(metaclass=Meta): + pass + class Derived(Base[int]): + pass + """) + self._assert_mro_matches_python(code, ["Derived"]) + + def test_subscripted_typing_generic_base_preserves_native_ancestor(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Generic, TypeVar + T = TypeVar("T") + class Foo(Generic[T]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Generic"],), namespace["Foo"].__bases__) + mod = libutils.get_string_module(self.project, code) + generic = self.project.get_module("typing")["Generic"].get_object() + foo = mod["Foo"].get_object() + self.assertIn(generic, foo.get_mro()) + + def test_generic_alias_omits_generic_before_direct_protocol(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Generic, Protocol, TypeVar + T = TypeVar("T") + class Derived(Generic[T], Protocol): + pass + """) + self._assert_mro_matches_python(code, ["Derived"]) + + def test_typing_protocol_specialization_preserves_native_ancestor(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Protocol, TypeVar + T = TypeVar("T") + class Derived(Protocol[T]): + pass + """) + self._assert_mro_matches_python(code, ["Derived"]) + + def test_typing_protocol_rejects_nonvariable_parameter(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Protocol + class Derived(Protocol[int]): + pass + """) + with self.assertRaises(TypeError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + with self.assertRaises(TypeError): + mod["Derived"].get_object().get_mro() + + def test_unpacked_typevartuple_preserves_generic_and_protocol_parameters(self): + self.project.prefs.set("import_dynload_stdmods", True) + available = all(name in typing.__dict__ for name in ("Unpack", "TypeVarTuple")) + for family in ("Generic", "Protocol"): + with self.subTest(family=family): + code = dedent(f"""\ + from typing import {family}, TypeVarTuple, Unpack + Ts = TypeVarTuple("Ts") + inner = Unpack[Ts] + alias = {family}[inner] + class Derived(alias): + pass + """) + namespace = {} + if available: + exec(code, namespace) + self.assertIs(namespace["inner"].__args__[0], namespace["Ts"]) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual( + [cls.__name__ for cls in namespace["Derived"].__mro__], + [cls.get_name() for cls in derived.get_mro()], + ) + inner = mod["inner"].get_object() + variable = ( + inner["__args__"] + .get_object() + .get_type() + .get_holding_objects()[0] + ) + self.assertIs(variable, mod["Ts"].get_object()) + self.assertIs(mod["alias"].get_object().arguments[0], inner) + else: + with self.assertRaises(ImportError): + exec(code, namespace) + mod = libutils.get_string_module(self.project, code) + for name in ("Unpack", "TypeVarTuple"): + if name not in typing.__dict__: + self.assertIs(mod[name].get_object(), get_unknown()) + with self.assertRaises(TypeError): + mod["Derived"].get_object().get_mro() + + def test_raw_typevartuple_is_not_a_generic_or_protocol_parameter(self): + self.project.prefs.set("import_dynload_stdmods", True) + for family in ("Generic", "Protocol"): + with self.subTest(family=family): + code = dedent(f"""\ + from typing import {family}, TypeVarTuple + Ts = TypeVarTuple("Ts") + class Derived({family}[Ts]): + pass + """) + if "TypeVarTuple" in typing.__dict__: + with self.assertRaises(TypeError): + exec(code, {}) + else: + with self.assertRaises(ImportError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + with self.assertRaises(TypeError): + mod["Derived"].get_object().get_mro() + + def test_named_typing_module_does_not_replace_its_declared_class_hook(self): + code = dedent("""\ + class Other: + pass + class Generic: + def __class_getitem__(cls, item): + return Other + class Derived(Generic[int]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + resource = testutils.create_module(self.project, "typing") + resource.write(code) + mod = self.project.get_pymodule(resource) + derived = mod["Derived"].get_object() + self.assertEqual([derived, mod["Other"].get_object()], derived.get_mro()) + + def test_generic_alias_omits_generic_before_later_typing_alias(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Generic, TypeVar + T = TypeVar("T") + class Base(Generic[T]): + pass + class Derived(Generic[T], Base[T]): + pass + """) + self._assert_mro_matches_python(code, ["Base", "Derived"]) + + def test_generic_alias_keeps_generic_before_builtin_alias(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Generic, TypeVar + T = TypeVar("T") + class Derived(Generic[T], list[T]): + pass + """) + self._assert_mro_matches_python(code, ["Derived"]) + + def test_generic_alias_rejects_repeated_parameters(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Generic, TypeVar + T = TypeVar("T") + class Derived(Generic[T, T]): + pass + """) + with self.assertRaises(TypeError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + with self.assertRaises(TypeError): + mod["Derived"].get_object().get_mro() + + def test_generic_alias_keeps_all_declared_parameters(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Generic, TypeVar + T = TypeVar("T") + U = TypeVar("U") + V = TypeVar("V") + W = TypeVar("W") + X = TypeVar("X") + alias = Generic[T, U, V, W, X] + class Derived(alias): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual(5, len(namespace["alias"].__args__)) + mod = libutils.get_string_module(self.project, code) + alias = mod["alias"].get_object() + self.assertEqual(5, len(alias.arguments)) + for name, argument in zip(("T", "U", "V", "W", "X"), alias.arguments): + self.assertIs(mod[name].get_object(), argument) + + def test_generic_alias_rejects_multiple_generic_declarations(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + from typing import Generic, TypeVar + T = TypeVar("T") + U = TypeVar("U") + class Derived(Generic[T], Generic[U]): + pass + """) + with self.assertRaises(TypeError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + with self.assertRaises(TypeError): + mod["Derived"].get_object().get_mro() + + def test_source_mro_entries_uses_original_bases_and_returned_class(self): + code = dedent("""\ + class Other: + pass + class Alias: + def __mro_entries__(self, bases): + return (bases[1],) + class Derived(Alias(), Other): + pass + """) + with self.assertRaises(TypeError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + with self.assertRaises(exceptions.InconsistentHierarchyError): + mod["Derived"].get_object().get_mro() + + def test_source_mro_entries_preserves_returned_class_identity(self): + code = dedent("""\ + class Other: + pass + class Alias: + def __mro_entries__(self, bases): + return (Other,) + class Derived(Alias()): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual([derived, mod["Other"].get_object()], derived.get_mro()) + + def test_source_mro_entries_preserves_every_returned_base(self): + for count in (4, 5): + with self.subTest(count=count): + names = "ABCDE"[:count] + code = dedent(f"""\ + class A(object): + pass + class B(object): + pass + class C(object): + pass + class D(object): + pass + class E(object): + pass + class Alias: + def __mro_entries__(self, bases): + return ({", ".join(names)},) + class Derived(Alias()): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual( + tuple(namespace[name] for name in names), + namespace["Derived"].__bases__, + ) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual( + [mod[name].get_object() for name in names], derived.get_bases() + ) + self.assertEqual( + [cls.__name__ for cls in namespace["Derived"].__mro__], + [cls.get_name() for cls in derived.get_mro()], + ) + + def test_source_mro_entries_rejects_a_noncallable_hook(self): + code = dedent("""\ + class Alias: + __mro_entries__ = None + class Derived(Alias()): + pass + """) + with self.assertRaises(TypeError): + exec(code, {}) + mod = libutils.get_string_module(self.project, code) + with self.assertRaises(TypeError): + mod["Derived"].get_object().get_mro() + + def test_source_mro_entries_staticmethod_receives_only_original_bases(self): + code = dedent("""\ + class Other: + pass + class Alias: + replacement = Other + @staticmethod + def __mro_entries__(bases): + return (bases[0].replacement,) + class Derived(Alias()): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual([derived, mod["Other"].get_object()], derived.get_mro()) + + def test_source_mro_entries_classmethod_receives_its_class(self): + code = dedent("""\ + class Alias: + @classmethod + def __mro_entries__(cls, bases): + return (cls,) + class Derived(Alias()): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Alias"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual([derived, mod["Alias"].get_object()], derived.get_mro()) + + def test_source_mro_entries_invokes_callable_instance(self): + code = dedent("""\ + class Other: + pass + class Rewriter: + def __call__(self, bases): + return (Other,) + class Alias: + __mro_entries__ = Rewriter() + class Derived(Alias()): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual([derived, mod["Other"].get_object()], derived.get_mro()) + + def test_source_mro_entries_class_lambda_retains_original_bases(self): + code = dedent("""\ + class Other: + pass + class Alias: + replacement = Other + __mro_entries__ = classmethod( + lambda cls, bases: (bases[0].replacement,) + ) + class Derived(Alias()): + pass + """) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace["Other"],), namespace["Derived"].__bases__) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual([derived, mod["Other"].get_object()], derived.get_mro()) + + @unittest.skipUnless(sys.version_info >= (3, 12), "PEP 695 requires Python 3.12") + def test_implicit_generic_specialization_preserves_owner_and_bound(self): + self.project.prefs.set("import_dynload_stdmods", True) + code = dedent("""\ + class Identity: + pass + class Ownership[Resource: Identity]: + pass + class Leaf(Ownership[Identity]): + pass + """) + namespace = {} + exec(code, namespace) + self.assertIs(namespace["Ownership"].__type_params__[0].__bound__, namespace["Identity"]) + mod = libutils.get_string_module(self.project, code) + owner = mod["Ownership"].get_object() + leaf = mod["Leaf"].get_object() + self.assertIs(leaf.get_mro()[1], owner) + parameter = owner.get_ast().type_params[0] + index = mod.get_source_binding_index() + binding = index.context_for_node(parameter).environment.lookup("Resource") + self.assertIs(binding.node, parameter) + bound = index.context_for_node(parameter.bound).environment.lookup("Identity") + self.assertIs(bound.get_object(), mod["Identity"].get_object()) + self.assertEqual( + [cls.__name__ for cls in namespace["Leaf"].__mro__], + [cls.get_name() for cls in leaf.get_mro()], + ) + + def test_mro_contains_object_only_when_declared(self): + code = dedent("""\ + class Base: + pass + class Derived(Base): + pass + """) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + self.assertEqual( + ["Derived", "Base"], [cls.get_name() for cls in derived.get_mro()] + ) + + def test_diamond_attributes_follow_the_mro(self): + code = dedent("""\ + class A(object): + def method(self): + pass + class B(A): + pass + class C(A): + def method(self): + pass + class D(B, C): + pass + """) + mod = libutils.get_string_module(self.project, code) + c = mod["C"].get_object() + d = mod["D"].get_object() + self.assertEqual(c["method"].get_object(), d["method"].get_object()) + + def test_unresolved_base_keeps_its_place(self): + code = dedent("""\ + class Base(object): + pass + class Other(object): + pass + class Derived(Base, NotFound, Other): + pass + """) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + other = mod["Other"].get_object() + derived = mod["Derived"].get_object() + first, unresolved, last = derived.get_bases() + self.assertEqual(base, first) + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertEqual("NotFound", unresolved.node.id) + self.assertIsNone(unresolved.pyobject) + self.assertEqual(other, last) + self.assertEqual([base, other], derived.get_superclasses()) + self.assertEqual( + [derived, base, unresolved, other, derived.get_mro()[-1]], + derived.get_mro(), + ) + self.assertEqual("object", derived.get_mro()[-1].get_name()) + + def test_base_evaluating_to_a_non_class_is_unresolved(self): + code = dedent("""\ + value = 1 + class Derived(value): + pass + """) + mod = libutils.get_string_module(self.project, code) + derived = mod["Derived"].get_object() + (unresolved,) = derived.get_bases() + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertEqual(mod["value"].get_object(), unresolved.pyobject) + self.assertEqual([], derived.get_superclasses()) + self.assertEqual([derived, unresolved], derived.get_mro()) + + def test_imported_missing_base_preserves_resolved_ancestors(self): + provider = testutils.create_module(self.project, "provider") + provider.write("def function():\n pass\n") + code = dedent("""\ + from provider import missing + class Base(object): + pass + class Derived(Base, missing): + pass + """) + mod = libutils.get_string_module(self.project, code) + base = mod["Base"].get_object() + derived = mod["Derived"].get_object() + mro = derived.get_mro() + resolved, unresolved = derived.get_bases() + self.assertIs(base, resolved) + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertEqual([base], derived.get_superclasses()) + self.assertEqual( + [derived, *base.get_mro(), unresolved], + mro, + ) + + def test_class_base_uses_import_before_rebinding_its_name(self): + provider = testutils.create_module(self.project, "provider") + provider.write(dedent("""\ + class Foo: + member = 1 + """)) + resource = testutils.create_module(self.project, "consumer") + resource.write(dedent("""\ + from provider import Foo + class Foo(Foo): + pass + """)) + imported = self.project.get_pymodule(provider)["Foo"].get_object() + derived = self.project.get_pymodule(resource)["Foo"].get_object() + self.assertEqual([derived, imported], derived.get_mro()) + self.assertIs(derived["member"], imported["member"]) + + def test_subscripted_base_uses_import_before_a_later_self_alias(self): + provider = testutils.create_module(self.project, "provider") + provider.write(dedent("""\ + class Service: + member = 1 + def __class_getitem__(cls, item): + return cls + s = Service + """)) + resource = testutils.create_module(self.project, "consumer") + resource.write(dedent("""\ + from provider import s + class Guards: + pass + class AppService(Guards, s[dict]): + pass + s = AppService + """)) + imported = self.project.get_pymodule(provider)["Service"].get_object() + consumer = self.project.get_pymodule(resource) + derived = consumer["AppService"].get_object() + guards = consumer["Guards"].get_object() + self.assertEqual([derived, guards, imported], derived.get_mro()) + self.assertIs(derived["member"], imported["member"]) + + def test_attribute_base_uses_module_before_a_later_rebinding(self): + provider = testutils.create_module(self.project, "provider") + provider.write(dedent("""\ + class Base: + member = 1 + """)) + resource = testutils.create_module(self.project, "consumer") + resource.write(dedent("""\ + import provider + class Derived(provider.Base): + pass + provider = Derived + """)) + imported = self.project.get_pymodule(provider)["Base"].get_object() + derived = self.project.get_pymodule(resource)["Derived"].get_object() + self.assertEqual([derived, imported], derived.get_mro()[:2]) + self.assertIs(derived["member"], imported["member"]) + + @unittest.skipUnless(sys.version_info >= (3, 12), "PEP 695 requires Python 3.12") + def test_generic_class_base_uses_import_before_rebinding_its_name(self): + self.project.prefs.set("import_dynload_stdmods", True) + provider = testutils.create_module(self.project, "provider") + provider.write(dedent("""\ + class Foo: + member = 1 + """)) + resource = testutils.create_module(self.project, "consumer") + resource.write(dedent("""\ + from provider import Foo + class Foo[T](Foo): + pass + """)) + imported = self.project.get_pymodule(provider)["Foo"].get_object() + derived = self.project.get_pymodule(resource)["Foo"].get_object() + namespace = {} + exec(provider.read(), namespace) + exec("class Foo[T](Foo):\n pass\n", namespace) + self.assertEqual( + [cls.__name__ for cls in namespace["Foo"].__mro__], + [cls.get_name() for cls in derived.get_mro()], + ) + self.assertIs(derived.get_mro()[1], imported) + self.assertIs(derived["member"], imported["member"]) + + @unittest.skipUnless(sys.version_info >= (3, 12), "PEP 695 requires Python 3.12") + def test_generic_parameter_precedes_imported_previous_class_name(self): + self.project.prefs.set("import_dynload_stdmods", True) + provider = testutils.create_module(self.project, "provider") + provider.write(dedent("""\ + class Foo: + pass + """)) + resource = testutils.create_module(self.project, "consumer") + resource.write(dedent("""\ + from provider import Foo + class Foo[Foo](Foo): + pass + """)) + derived = self.project.get_pymodule(resource)["Foo"].get_object() + unresolved, generic = derived.get_bases() + self.assertIsInstance(unresolved, UnresolvedBase) + self.assertIs(unresolved.pyobject, get_unknown()) + self.assertIs(self.project.get_module("typing")["Generic"].get_object(), generic) + + def test_inconsistent_mro(self): + code = dedent("""\ + class A(object): + pass + class B(object): + pass + class X(A, B): + pass + class Y(B, A): + pass + class Z(X, Y): + pass + class W(Z): + pass + """) + mod = libutils.get_string_module(self.project, code) + z = mod["Z"].get_object() + with self.assertRaises(exceptions.InconsistentHierarchyError): + z.get_mro() + self.assertIsNone(z.get_consistent_mro()) + self.assertIsNone(mod["W"].get_object().get_consistent_mro()) + + def test_inconsistent_mro_keeps_attributes_along_the_bases(self): + # Python refuses to create these classes, but rope keeps analysing + # them as it always did: only `get_mro()` reports the hierarchy. + code = dedent("""\ + class A(object): + def a_method(self): + pass + class B(A): + def b_method(self): + pass + class C(A, B): + def c_method(self): + pass + """) + mod = libutils.get_string_module(self.project, code) + c = mod["C"].get_object() + self.assertEqual( + {"a_method", "b_method", "c_method"}, + {name for name in c.get_attributes() if name.endswith("_method")}, + ) + self.assertEqual( + mod["B"].get_object()["b_method"].get_object(), + c["b_method"].get_object(), + ) + self.assertEqual( + [mod["A"].get_object(), mod["B"].get_object()], c.get_superclasses() + ) + + def test_type_hinting_get_mro_walks_the_bases(self): + from rope.base.oi.type_hinting.utils import get_mro + + code = dedent("""\ + class A(object): + pass + class B(A): + pass + class C(B, A): + pass + """) + mod = libutils.get_string_module(self.project, code) + a, b, c = (mod[name].get_object() for name in ("A", "B", "C")) + self.assertEqual([c, b, a], get_mro(c)) + + def test_duplicate_base_class(self): + code = dedent("""\ + class A(object): + pass + class B(A, A): + pass + """) + mod = libutils.get_string_module(self.project, code) + with self.assertRaises(exceptions.InconsistentHierarchyError): + mod["B"].get_object().get_mro() + self.assertIsNone(mod["B"].get_object().get_consistent_mro()) + + def test_cyclic_inheritance(self): + mod1 = testutils.create_module(self.project, "mod1") + mod2 = testutils.create_module(self.project, "mod2") + mod1.write("from mod2 import B\nclass A(B):\n pass\n") + mod2.write("from mod1 import A\nclass B(A):\n pass\n") + a = self.project.get_module("mod1")["A"].get_object() + with self.assertRaises(exceptions.InconsistentHierarchyError): + a.get_mro() + self.assertIsNone(a.get_consistent_mro()) + + def test_offset_resolution_follows_changes_in_other_modules(self): + mod1 = testutils.create_module(self.project, "mod1") + mod2 = testutils.create_module(self.project, "mod2") + mod2.write("class C:\n attr = 1\n") + code = "from mod2 import C\nresult = C().attr\n" + mod1.write(code) + offset = code.rindex("attr") + + def attribute_definition(): + pymodule = self.project.get_pymodule(mod1) + pyname = evaluate.eval_location(pymodule, offset) + return pyname.get_definition_location() + + module, lineno = attribute_definition() + self.assertEqual((self.project.get_pymodule(mod2), 2), (module, lineno)) + mod2.write("class C:\n\n attr = 1\n") + module, lineno = attribute_definition() + self.assertEqual((self.project.get_pymodule(mod2), 3), (module, lineno)) + def test_module_creation(self): new_module = testutils.create_module(self.project, "module") self.assertFalse(new_module.is_folder()) @@ -330,6 +1650,100 @@ def test_find_module_when_module_and_package_with_the_same_name(self): found_module = self.project.find_module("sample") self.assertEqual(samplepkg, found_module) + def test_find_module_prefers_module_to_namespace_directory(self): + self.project.root.create_folder("sample") + samplemod = testutils.create_module(self.project, "sample") + self.assertEqual(samplemod, self.project.find_module("sample")) + + def test_find_module_retains_namespace_directory(self): + namespace = self.project.root.create_folder("sample") + self.project.close() + self.project = Project(self.project.address, source_folders=[""]) + self.assertEqual(namespace, self.project.find_module("sample")) + + def test_find_nested_module_does_not_search_shadowed_namespace(self): + namespace = self.project.root.create_folder("sample") + testutils.create_module(self.project, "child", namespace) + testutils.create_module(self.project, "sample") + self.assertIsNone(self.project.find_module("sample.child")) + + def test_find_module_skips_python_path_files(self): + external = tempfile.TemporaryDirectory() + self.addCleanup(external.cleanup) + # Rope reports real paths; temporary directories may sit behind a + # symlink (macOS /var) or a short name (Windows RUNNER~1). + root = os.path.realpath(external.name) + archive = os.path.join(root, "archive.zip") + libs = os.path.join(root, "libs") + os.mkdir(libs) + open(archive, "w").close() + with open(os.path.join(libs, "sample.py"), "w") as sample: + sample.write("") + self.project.close() + self.project = Project(self.project.address, python_path=[archive, libs]) + self.assertEqual( + os.path.join(libs, "sample.py"), + self.project.find_module("sample").real_path, + ) + + def test_find_module_defers_namespace_to_later_source_module(self): + first = self.project.root.create_folder("first") + second = self.project.root.create_folder("second") + first.create_folder("sample") + samplemod = testutils.create_module(self.project, "sample", second) + self.project.close() + self.project = Project(self.project.address, source_folders=["first", "second"]) + self.assertEqual(samplemod, self.project.find_module("sample")) + + def test_find_module_defers_namespace_to_later_regular_package(self): + first = self.project.root.create_folder("first") + second = self.project.root.create_folder("second") + namespace = first.create_folder("sample") + testutils.create_module(self.project, "child", namespace) + package = testutils.create_package(self.project, "sample", second) + self.project.close() + self.project = Project(self.project.address, source_folders=["first", "second"]) + self.assertEqual(package, self.project.find_module("sample")) + self.assertIsNone(self.project.find_module("sample.child")) + + def test_find_nested_module_searches_namespace_portions(self): + first = self.project.root.create_folder("first") + second = self.project.root.create_folder("second") + first.create_folder("sample") + namespace = second.create_folder("sample") + samplemod = testutils.create_module(self.project, "child", namespace) + self.project.close() + self.project = Project(self.project.address, source_folders=["first", "second"]) + self.assertEqual(samplemod, self.project.find_module("sample.child")) + + def test_relative_import_prefers_config_module_to_namespace_directory(self): + package = testutils.create_package(self.project, "sample") + package.create_folder("config") + config = testutils.create_module(self.project, "config", package) + config.write("class FlextCliConfig:\n pass\n") + consumer = testutils.create_module(self.project, "consumer", package) + consumer.write("from .config import FlextCliConfig\n") + expected = self.project.get_pymodule(config)["FlextCliConfig"].get_object() + actual = self.project.get_pymodule(consumer)["FlextCliConfig"].get_object() + self.assertEqual(expected, actual) + + def test_relative_import_keeps_regular_package_precedence(self): + package = testutils.create_package(self.project, "sample") + testutils.create_module(self.project, "config", package) + config = testutils.create_package(self.project, "config", package) + config.get_child("__init__.py").write("class Configuration:\n pass\n") + consumer = testutils.create_module(self.project, "consumer", package) + consumer.write("from .config import Configuration\n") + expected = self.project.get_pymodule(config)["Configuration"].get_object() + actual = self.project.get_pymodule(consumer)["Configuration"].get_object() + self.assertEqual(expected, actual) + + def test_relative_module_does_not_search_unrelated_source_roots(self): + package = testutils.create_package(self.project, "sample") + testutils.create_module(self.project, "config") + with self.assertRaises(exceptions.ModuleNotFoundError): + self.project.get_relative_module("config", package, 1) + def test_source_folders_preference(self): testutils.create_package(self.project, "pkg1") testutils.create_package(self.project, "pkg1.src2") diff --git a/ropetest/pyscopestest.py b/ropetest/pyscopestest.py index d60702b32..6edd11f40 100644 --- a/ropetest/pyscopestest.py +++ b/ropetest/pyscopestest.py @@ -1,8 +1,9 @@ import unittest from textwrap import dedent -from rope.base import libutils +from rope.base import exceptions, libutils from rope.base.pyobjects import get_base_type +from rope.refactor import patchedast from ropetest import testutils @@ -26,6 +27,118 @@ def sample_func(): sample_func = scope["sample_func"].get_object() self.assertEqual(get_base_type("Function"), sample_func.get_type()) + def test_indexed_absolute_import_roots_preserve_all_declarations(self): + source = dedent('''\ + import outer.first as replaced, second + replaced = None + from absolute.child import * + from .relative import ignored + if False: + from guarded.child import value + class Owner: + import class_scope.child + def method(self): + import function_scope.child as local + from ..relative import hidden + text = "import not_a_declaration" + ''') + module = libutils.get_string_module(self.project, source) + index = module.get_source_binding_index() + expected = frozenset( + {"outer", "second", "absolute", "guarded", "class_scope", "function_scope"} + ) + self.assertEqual(expected, index.absolute_import_roots()) + self.assertIs(index, module.get_source_binding_index()) + self.assertEqual(expected, index.absolute_import_roots()) + + def test_indexed_absolute_import_roots_empty_module(self): + module = libutils.get_string_module(self.project, "value = 1\n") + self.assertEqual(frozenset(), module.get_source_binding_index().absolute_import_roots()) + + def test_indexed_assignment_string_sequences(self): + cases = ( + ("value = 1\n", None), + ("names: tuple[str, ...]\n", ()), + ("names = ('first',)\nnames = ('second',)\n", ("second",)), + ("names = ('one',)\nnames: tuple[str, ...]\n", ("one",)), + ("names = ('one',)\nnames += ['two']\n", ("one", "two")), + ("names = ('one',)\nnames += other\n", ()), + ("names = ('one',)\nnames = ('two',)\nnames += ('three',)\n", ("two", "three")), + ( + "shared = ('one',)\nshared += ('two',)\n" + "names = tuple(shared)\nnames += ('three',)\n", + ("one", "two", "three"), + ), + ("alias = names = ['one', 'two']\n", ("one", "two")), + ("names: tuple[str, ...] = ('é',)\n", ("é",)), + ("shared = ('one',)\nnames = tuple(shared)\n", ("one",)), + ("shared = ('one',)\nnames = tuple(shared)\nshared += ('two',)\n", ("one",)), + ("shared = ('one',); names = shared; shared = ('two',)\n", ("one",)), + ("names = shared\nshared = ('later',)\n", ()), + ("names = []\n", ()), + ("names = ['one', 2]\n", ()), + ("names = [*other]\n", ()), + ("if False:\n names = ('guarded',)\n", None), + ) + for source, expected in cases: + with self.subTest(source=source): + module = libutils.get_string_module(self.project, source) + self.assertEqual( + expected, + module.get_source_binding_index().assignment_string_sequence("names"), + ) + + def test_indexed_assignment_sequence_class_scope_and_module_reference(self): + module = libutils.get_string_module( + self.project, "shared = ('value',)\nclass Owner:\n names = shared\n" + ) + index = module.get_source_binding_index() + self.assertIsNone(index.assignment_string_sequence("names")) + owner_scope = module["Owner"].get_object().get_scope() + self.assertEqual(("value",), index.assignment_string_sequence("names", owner_scope)) + + def test_indexed_assignment_sequence_cycles_fail(self): + module = libutils.get_string_module(self.project, "names = other\nother = names\n") + with self.assertRaisesRegex(ValueError, "Cyclic"): + module.get_source_binding_index().assignment_string_sequence("names") + + def test_indexed_assignment_sequence_rejects_foreign_scope(self): + local = libutils.get_string_module(self.project, "names = ('local',)\n") + foreign = libutils.get_string_module(self.project, "class Owner:\n names = ('foreign',)\n") + scope = foreign["Owner"].get_object().get_scope() + with self.assertRaisesRegex(ValueError, "another source module"): + local.get_source_binding_index().assignment_string_sequence("names", scope) + + def test_indexed_assignment_sequence_class_metadata_precedence(self): + cases = ( + ("class Owner:\n names = ('class',)\n", ("class",)), + ("names = []\nclass Owner:\n names = ('class',)\n", ()), + ("names = 1\nclass Owner:\n names = ('class',)\n", ()), + ("class First: pass\nclass Second:\n names = ('class',)\n", None), + ("class Owner: pass\n", None), + ("class Owner:\n names = names\n", ()), + ("if True:\n class Owner:\n names = ('guarded',)\n", None), + ) + for source, expected in cases: + with self.subTest(source=source): + module = libutils.get_string_module(self.project, source) + self.assertEqual( + expected, + module.get_source_binding_index().assignment_string_sequence( + "names", class_metadata=True + ), + ) + + def test_indexed_assignment_sequence_same_name_in_module_and_class(self): + module = libutils.get_string_module( + self.project, "names = ('module',)\nclass Owner:\n names = names\n" + ) + scope = module["Owner"].get_object().get_scope() + self.assertEqual( + ("module",), + module.get_source_binding_index().assignment_string_sequence("names", scope), + ) + def test_simple_function_scope(self): code = dedent("""\ def sample_func(): @@ -112,6 +225,25 @@ def test_inline_assignment(self): ["a_var", "values"], ) + @testutils.only_for_versions_higher("3.10") + def test_match_capture_names(self): + code = dedent("""\ + def func(value): + match value: + case [_, *rest] as whole: + pass + case {"key": item, **remaining}: + pass + """) + scope = libutils.get_string_scope(self.project, code) + function_scope = scope.get_scopes()[0] + defined_names = set(function_scope.get_names()) + + self.assertLessEqual( + {"item", "remaining", "rest", "whole"}, defined_names + ) + self.assertNotIn("_", defined_names) + @testutils.only_for_versions_higher("3.8") def test_inline_assignment_in_comprehensions(self): code = dedent("""\ @@ -560,3 +692,186 @@ def func2(o): ) self.assertEqual(scope.get_scopes()[1].get_region(), (26, 47)) + + def test_decorated_scope_region_is_stable_across_calls(self): + scope = libutils.get_string_scope( + self.project, + dedent("""\ + @staticmethod + def func(ala): + pass + """), + ) + inner = scope.get_scopes()[0] + + self.assertEqual(inner.get_region(), (0, 37)) + self.assertEqual(inner.get_region(), inner.get_region()) + + def test_scope_region_follows_changed_module_source(self): + mod = testutils.create_module(self.project, "mod") + mod.write("def func():\n pass\n") + first = self.project.get_pymodule(mod).get_scope().get_scopes()[0] + self.assertEqual(first.get_region(), (0, 20)) + + mod.write("\n\ndef func():\n return 1\n") + second = self.project.get_pymodule(mod).get_scope().get_scopes()[0] + + self.assertEqual(second.get_region(), (2, 26)) + + def test_scope_regions_with_normalized_import_alias(self): + alias = "u\u0301" + code = ( + f"from os import path as {alias}\n" + f"def outer({alias}):\n" + " def inner():\n" + f" marker = 'π'; return {alias}.sep\n" + " return inner\n" + ) + scope = libutils.get_string_scope(self.project, code) + outer = scope.get_scopes()[0] + inner = outer.get_scopes()[0] + + self.assertIs(scope, scope.get_inner_scope_for_offset(code.index(alias))) + self.assertIs( + inner, scope.get_inner_scope_for_offset(code.index(alias + ".sep")) + ) + self.assertIs(inner.lookup("ú"), outer.lookup("ú")) + self.assertIsNot(inner.lookup("ú"), scope.lookup("ú")) + + def test_decorated_scope_regions_preserve_raw_string_newlines(self): + for newline in ("\n", "\r\n", "\r"): + for definition in ( + "def sample(value):\n marker = 'π'; return value\n", + "async def sample(value):\n marker = 'π'; return value\n", + "class Sample:\n marker = 'π'; value = 1\n", + ): + with self.subTest(newline=newline, definition=definition): + code = ( + "from os import path as u\u0301\n" + "@(\n" + " # @ is not the decorator marker\n" + " decorate\n" + ")\n" + "@other\n" + + definition + ).replace("\n", newline) + module = libutils.get_string_module(self.project, code) + self.assertEqual(code, module.source_code) + scope = module.get_scope() + child = scope.get_scopes()[0] + expected = (code.index("@("), len(code.rstrip("\r\n"))) + + self.assertEqual((0, len(code)), scope.get_region()) + self.assertEqual(expected, child.get_region()) + self.assertIs( + child, + scope.get_inner_scope_for_offset(code.index("decorate") + 1), + ) + self.assertFalse(child.in_region(expected[0])) + self.assertTrue(child.in_region(expected[0] + 1)) + self.assertTrue(child.in_region(expected[1] - 1)) + self.assertFalse(child.in_region(expected[1])) + + def test_scope_regions_use_the_decoded_resource_source(self): + for index, newline in enumerate(("\n", "\r\n", "\r")): + with self.subTest(newline=newline): + resource = testutils.create_module(self.project, "regions_%s" % index) + resource.write( + "@(\n decorate\n)\ndef sample():\n return 'π'\n".replace( + "\n", newline + ) + ) + module = self.project.get_pymodule(resource) + source = module.source_code + self.assertNotIn("\r", source) + scope = module.get_scope() + + self.assertEqual((0, len(source)), scope.get_region()) + self.assertEqual( + (source.index("@("), len(source.rstrip("\n"))), + scope.get_scopes()[0].get_region(), + ) + + def test_scope_regions_with_normalized_alias_in_comprehensions(self): + alias = "u\u0301" + expressions = ( + f"[({alias}, item) for item in values]", + f"{{({alias}, item) for item in values}}", + f"{{item: {alias} for item in values}}", + f"(({alias}, item) for item in values)", + ) + for expression in expressions: + with self.subTest(expression=expression): + code = ( + f"from os import path as {alias}\n" + "def sample(values):\n" + f" result = {expression}\n" + " return result\n" + ) + scope = libutils.get_string_scope(self.project, code) + function = scope.get_scopes()[0] + comprehension = function.get_scopes()[0] + + self.assertIs( + comprehension, + scope.get_inner_scope_for_offset(code.rindex(alias)), + ) + self.assertIs(comprehension.lookup("ú"), scope.lookup("ú")) + + def test_scope_region_does_not_partially_patch_the_module_ast(self): + code = "def sample(value):\n return value\n" + module = libutils.get_string_module(self.project, code) + module.get_scope().get_scopes()[0].get_region() + patched = patchedast.patch_ast(module.get_ast(), code, sorted_children=True) + + self.assertEqual(code, patchedast.write_ast(patched)) + + def test_scope_region_rejects_incomplete_source_positions(self): + scope = libutils.get_string_scope(self.project, "def sample(): pass\n") + child = scope.get_scopes()[0] + child.pyobject.get_ast().end_lineno = None + + with self.assertRaisesRegex(exceptions.RopeError, "complete source positions"): + child.get_region() + + def test_scope_region_rejects_decorator_without_source_positions(self): + code = "@decorator\ndef sample(): pass\n" + scope = libutils.get_string_scope(self.project, code) + child = scope.get_scopes()[0] + child.pyobject.get_ast().decorator_list[0].end_lineno = None + + with self.assertRaisesRegex(exceptions.RopeError, "Decorator AST"): + child.get_region() + + def test_scope_region_rejects_decorator_without_source_marker(self): + code = "@decorator\ndef sample(): pass\n" + scope = libutils.get_string_scope(self.project, code) + child = scope.get_scopes()[0] + decorator = child.pyobject.get_ast().decorator_list[0] + decorator.col_offset = 0 + decorator.end_col_offset = 0 + + with self.assertRaisesRegex(exceptions.RopeError, "source marker"): + child.get_region() + + def test_scope_decorator_marker_uses_python_line_boundaries(self): + for prefix in ("\f\n", "# leading\vcomment\n"): + with self.subTest(prefix=prefix): + code = ( + prefix + + "@(\n" + " # @ is not the decorator marker\n" + " decorate\n" + ")\n" + "def sample():\n" + " pass\n" + ) + scope = libutils.get_string_scope(self.project, code) + child = scope.get_scopes()[0] + + self.assertEqual( + (code.index("@("), len(code.rstrip("\n"))), child.get_region() + ) + self.assertIs( + child, scope.get_inner_scope_for_offset(code.index("decorate")) + ) diff --git a/ropetest/refactor/__init__.py b/ropetest/refactor/__init__.py index 4d5bee1ad..a3e116ba6 100644 --- a/ropetest/refactor/__init__.py +++ b/ropetest/refactor/__init__.py @@ -1,6 +1,8 @@ import unittest from textwrap import dedent +import pytest + import rope.base.taskhandle import rope.refactor.introduce_parameter import ropetest.refactor.extracttest @@ -19,7 +21,8 @@ from rope.refactor.localtofield import LocalToField from rope.refactor.method_object import MethodObject from ropetest import testutils -from ropetest.refactor import change_signature_test, similarfindertest +from ropetest.refactor import change_signature_test as change_signature_test +from ropetest.refactor import similarfindertest as similarfindertest class MethodObjectTest(unittest.TestCase): @@ -1118,14 +1121,16 @@ def test_job_set_get_percent_done(self): def test_getting_job_name(self): handle = rope.base.taskhandle.TaskHandle() jobs = handle.create_jobset(name="test job set", count=1) - # recommended name/job_name attribute - self.assertEqual("test job set", jobs.name) - self.assertEqual(None, jobs.job_name) - # deprecated getters - self.assertEqual("test job set", jobs.get_name()) - self.assertEqual(None, jobs.get_active_job_name()) - jobs.started_job("job1") - # recommended name/job_name attribute - self.assertEqual("test job set", jobs.get_name()) - # deprecated getters - self.assertEqual("job1", jobs.get_active_job_name()) + for job_name in (None, "job1"): + if job_name is not None: + jobs.started_job(job_name) + self.assertEqual("test job set", jobs.name) + self.assertEqual(job_name, jobs.job_name) + for getter, attribute, expected in ( + (jobs.get_name, "name", "test job set"), + (jobs.get_active_job_name, "job_name", job_name), + ): + message = rf"^Just use JobSet\.{attribute} attribute/property instead$" + with pytest.warns(DeprecationWarning, match=message) as recorded: + self.assertEqual(expected, getter()) + self.assertEqual(1, len(recorded)) diff --git a/ropetest/refactor/change_signature_test.py b/ropetest/refactor/change_signature_test.py index 195e15223..240640475 100644 --- a/ropetest/refactor/change_signature_test.py +++ b/ropetest/refactor/change_signature_test.py @@ -7,6 +7,75 @@ class ChangeSignatureTest(unittest.TestCase): + def test_removing_argument_preserves_parameter_annotations(self): + code = dedent("""\ + def keep(value: tuple[str, int], unused: object, *, flag: bool = False) -> tuple[str, int]: + return value + result = keep(("kept", 1), object(), flag=True) + """) + self.mod.write(code) + signature = change_signature.ChangeSignature( + self.project, self.mod, code.index("keep") + ) + with self.assertRaises(rope.base.exceptions.RefactoringError): + signature.get_changes([change_signature.ArgumentRemover(1)]) + self.assertEqual(code, self.mod.read()) + + def test_removing_argument_retains_supported_parameter_annotations(self): + code = dedent("""\ + def keep(value: tuple[str, int], unused: object, flag: bool = False) -> tuple[str, int]: + return value + result = keep(("kept", 1), object(), flag=True) + """) + self.mod.write(code) + signature = change_signature.ChangeSignature( + self.project, self.mod, code.index("keep") + ) + self.project.do(signature.get_changes([change_signature.ArgumentRemover(1)])) + namespace = {} + exec(self.mod.read(), namespace) + self.assertEqual(("kept", 1), namespace["result"]) + self.assertEqual( + {"value": tuple[str, int], "flag": bool, "return": tuple[str, int]}, + namespace["keep"].__annotations__, + ) + + def test_removing_argument_retains_unicode_source_spans(self): + code = dedent('''\ + def keep(café: str, unused: object, next: tuple[str, int]) -> str: + return café + result = keep("ação", object(), ("próximo", 1)) + ''') + self.mod.write(code) + signature = change_signature.ChangeSignature(self.project, self.mod, code.index("keep")) + self.project.do(signature.get_changes([change_signature.ArgumentRemover(1)])) + namespace = {} + exec(self.mod.read(), namespace) + self.assertEqual("ação", namespace["result"]) + self.assertEqual( + {"café": str, "next": tuple[str, int], "return": str}, + namespace["keep"].__annotations__, + ) + + def test_removing_argument_retains_defaults_and_variadic_annotations(self): + code = dedent('''\ + def keep(value: int = 1, unused: object = None, *rest: str, **options: int) -> tuple: + return value, rest, options + result = keep(7, None, "ação", count=2) + defaults = keep() + ''') + self.mod.write(code) + signature = change_signature.ChangeSignature(self.project, self.mod, code.index("keep")) + self.project.do(signature.get_changes([change_signature.ArgumentRemover(1)])) + namespace = {} + exec(self.mod.read(), namespace) + self.assertEqual((7, ("ação",), {"count": 2}), namespace["result"]) + self.assertEqual((1, (), {}), namespace["defaults"]) + self.assertEqual( + {"value": int, "rest": str, "options": int, "return": tuple}, + namespace["keep"].__annotations__, + ) + def setUp(self): super().setUp() self.project = testutils.sample_project() @@ -811,6 +880,98 @@ def __init__(self, p): self.mod.read(), ) + def test_changing_signature_for_function_local_constructor(self): + code = dedent("""\ + def make(): + class Local: + def __init__(self, value): + pass + item = Local(1) + item.__init__(2) + return item + """) + expected = dedent("""\ + def make(): + class Local: + def __init__(self): + pass + item = Local() + item.__init__() + return item + """) + offsets = ( + code.index("Local"), + code.index("__init__"), + code.rindex("Local"), + code.rindex("__init__"), + ) + for offset in offsets: + with self.subTest(offset=offset): + self.mod.write(code) + signature = change_signature.ChangeSignature( + self.project, self.mod, offset + ) + self.project.do( + signature.get_changes([change_signature.ArgumentRemover(1)]) + ) + self.assertEqual(expected, self.mod.read()) + + def test_function_local_constructors_with_the_same_name_are_distinct(self): + self.mod.write(dedent("""\ + def first(): + class Local: + def __init__(self, value): + pass + return Local(1) + def second(): + class Local: + def __init__(self, value): + pass + return Local(2) + """)) + signature = change_signature.ChangeSignature( + self.project, self.mod, self.mod.read().index("__init__") + ) + self.project.do(signature.get_changes([change_signature.ArgumentRemover(1)])) + self.assertEqual( + dedent("""\ + def first(): + class Local: + def __init__(self): + pass + return Local() + def second(): + class Local: + def __init__(self, value): + pass + return Local(2) + """), + self.mod.read(), + ) + + def test_changing_signature_for_class_nested_constructor(self): + self.mod.write(dedent("""\ + class Outer: + class Inner: + def __init__(self, value): + pass + item = Outer.Inner(1) + """)) + signature = change_signature.ChangeSignature( + self.project, self.mod, self.mod.read().index("__init__") + ) + self.project.do(signature.get_changes([change_signature.ArgumentRemover(1)])) + self.assertEqual( + dedent("""\ + class Outer: + class Inner: + def __init__(self): + pass + item = Outer.Inner() + """), + self.mod.read(), + ) + def test_redordering_arguments_reported_by_mft(self): self.mod.write(dedent("""\ def f(a, b, c): diff --git a/ropetest/refactor/class_function_access_test.py b/ropetest/refactor/class_function_access_test.py new file mode 100644 index 000000000..ff2898591 --- /dev/null +++ b/ropetest/refactor/class_function_access_test.py @@ -0,0 +1,171 @@ +import os +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base import arguments, evaluate, libutils, pyobjects +from rope.base.project import Project +from rope.refactor.rename import Rename + +CLASSES = dedent("""\ + class Owner: + def __init__(self): + self.field = 1 + callback = module_callback + class Child(Owner): + def __init__(self): + self.field = 3 + class Other: + def __init__(self): + self.field = 2 +""") +PREFS = dict( + save_history=False, + perform_doa=False, + automatic_soa=False, +) + + +def _write_inputs(directory, helper, subject): + directory.mkdir() + (directory / "helper.py").write_text(helper, encoding="utf-8") + (directory / "subject.py").write_text(subject, encoding="utf-8") + (directory / "sentinel.txt").write_bytes(b"untouched\n") + return _input_bytes(directory) + + +def _input_bytes(directory): + return { + name: (directory / name).read_bytes() + for name in ("helper.py", "subject.py", "sentinel.txt") + } + + +def _run(directory): + env = os.environ.copy() + env.pop("PYTHONPATH", None) + result = subprocess.run( + [sys.executable, "-B", "subject.py"], + cwd=directory, + env=env, + capture_output=True, + text=True, + timeout=45, + ) + return result.returncode, result.stdout, result.stderr + + +@pytest.mark.parametrize("access", ["direct", "saved"]) +def test_imported_inherited_function_parameter_rename(tmp_path, access): + helper_code = dedent("""\ + def module_callback(self, item): + return item + """) + call = ( + "result = Child().callback(item=Other())\n" + if access == "direct" + else "saved = Child().callback\nresult = saved(item=Other())\n" + ) + subject_code = ( + "from helper import module_callback\n" + + CLASSES + + call + + "print(result.field)\n" + ) + directory = tmp_path / "project" + original = _write_inputs(directory, helper_code, subject_code) + before = _run(directory) + assert before == (0, "2\n", "") + project = Project(directory, ropefolder=None, save_objectdb=False, **PREFS) + try: + helper = project.get_file("helper.py") + resource = project.get_file("subject.py") + offset = helper.read().index("item") + 1 + changes = Rename(project, helper, offset).get_changes("renamed") + assert _input_bytes(directory) == original + project.do(changes) + after_helper, after_subject = helper.read(), resource.read() + assert "def module_callback(self, renamed):" in after_helper + assert "return renamed" in after_helper + assert "renamed=Other()" in after_subject + assert _input_bytes(directory) != original + assert _run(directory) == before + assert (directory / "sentinel.txt").read_bytes() == original["sentinel.txt"] + + helper_module = project.get_pymodule(helper) + subject_module = project.get_pymodule(resource) + function = helper_module.get_scope().lookup("module_callback").get_object() + keyword = evaluate.eval_location( + subject_module, after_subject.index("renamed=Other") + 1 + ) + assert keyword is function.get_parameters()["renamed"] + assert function.parent is helper_module + assert function.get_module() is helper_module + owner = subject_module.get_scope().lookup("Owner").get_object() + slot = evaluate.eval_location( + subject_module, after_subject.index("callback = module_callback") + 1 + ) + assert slot is owner["callback"] + assert slot.get_definition_location()[0] is subject_module + finally: + project.close() + + +def test_imported_inherited_callback_survives_analysis_and_reopen(tmp_path): + helper_code = dedent("""\ + def module_callback(self, item): + return self + """) + use_code = dedent("""\ + def use(callback, item): + return callback(item) + """) + calls = dedent("""\ + direct = Child().callback(Other()) + saved = Child().callback + result = use(saved, Other()) + print(result.field) + """) + subject_code = "from helper import module_callback\n" + use_code + CLASSES + calls + directory = tmp_path / "project" + original = _write_inputs(directory, helper_code, subject_code) + before = _run(directory) + assert before == (0, "3\n", "") + + for analyze in (True, False): + project = Project( + directory, save_objectdb=True, validate_objectdb=True, **PREFS + ) + try: + resource = project.get_file("subject.py") + helper = project.get_file("helper.py") + if analyze: + libutils.analyze_module(project, resource) + subject_module = project.get_pymodule(resource) + helper_module = project.get_pymodule(helper) + scope = subject_module.get_scope() + child = scope.lookup("Child").get_object() + function = helper_module.get_scope().lookup("module_callback").get_object() + use = scope.lookup("use").get_object() + callback_values = project.pycore.object_info.get_passed_objects(use, 0) + assert callback_values + for callback in callback_values: + assert isinstance(callback, pyobjects.AbstractFunction) + assert callback.get_name() == function.get_name() + returned = callback.get_returned_object( + arguments.ObjectArguments([scope.lookup("Other")]) + ) + assert returned.get_type() is child + receivers = project.pycore.object_info.get_passed_objects(function, 0) + assert any(receiver.get_type() is child for receiver in receivers) + assert scope.lookup("result").get_object().get_type() is child + assert function.parent is helper_module + assert function.get_module() is helper_module + assert _input_bytes(directory) == original + assert _run(directory) == before + finally: + project.close() + assert any((directory / ".ropeproject").iterdir()) + assert _input_bytes(directory) == original diff --git a/ropetest/refactor/class_stored_function_regression_test.py b/ropetest/refactor/class_stored_function_regression_test.py new file mode 100644 index 000000000..ca3c5efa8 --- /dev/null +++ b/ropetest/refactor/class_stored_function_regression_test.py @@ -0,0 +1,236 @@ +import json + +import pytest + +from rope.base import pyobjects +from rope.refactor.rename import Rename +from ropetest.refactor.class_stored_function_test import ( + CLASS_ALIAS, + MODULE_FUNCTION, + OTHER, + OWNER, + _project, + _run, +) + +PREFIX = MODULE_FUNCTION + OWNER + CLASS_ALIAS + OTHER +INHERITED = ( + PREFIX + + "class Child(Owner):\n def __init__(self):\n self.field = 3\n" + + "result = Child().callback(Other())\nprint(result.field)\n" +) +CASES = { + "alias_override": ( + PREFIX + + "instance = Owner()\nalias = instance\nalias.callback = module_callback\n" + + "result = instance.callback(Other(), Owner())\nprint(result.field)\n", + "Other", + "2\n", + ), + "inherited": (INHERITED, "Child", "3\n"), + "object_base": ( + PREFIX.replace("class Owner:", "class Owner(object):") + + "result = Owner().callback(Other())\nprint(result.field)\n", + "Owner", + "1\n", + ), + "different_instance_override": ( + PREFIX + + "first = Owner()\nsecond = Owner()\nsecond.callback = module_callback\n" + + "result = first.callback(Other())\nprint(result.field)\n", + "Owner", + "1\n", + ), +} + + +def _query(directory, code, expected_class): + project, resource = _project(directory, code) + try: + scope = project.get_pymodule(resource).get_scope() + result = scope.lookup("result").get_object() + return { + "unknown": result is pyobjects.get_unknown(), + "type_expected": result.get_type() + is scope.lookup(expected_class).get_object(), + } + finally: + project.close() + + +def _exercise(tmp_path, fixture, code, expected_class, expected_stdout, selected): + query = _query(tmp_path / "query", code, expected_class) + directory = tmp_path / "cold" + project, resource = _project(directory, code) + original = {p.name: p.read_bytes() for p in directory.iterdir() if p.is_file()} + if selected == "parent_slot": + offset = code.index(" callback =") + len(" ") + 1 + new_name = "renamed_callback" + else: + value = {"Owner": "1", "Other": "2", "Child": "3"}[selected] + offset = code.index("self.field = " + value) + len("self.") + 1 + new_name = "changed" + evidence = { + "fixture": fixture, + "selected": selected, + "query": query, + "before_source": code, + "offset": offset, + "new_name": new_name, + "query_project_separate_from_cold": True, + "original_bytes_hex": {name: data.hex() for name, data in original.items()}, + } + try: + before = _run(code, directory) + evidence["before"] = before + assert before == (0, expected_stdout, "") + changes = Rename(project, resource, offset).get_changes(new_name) + assert original == {name: (directory / name).read_bytes() for name in original} + evidence["changeset"] = changes.get_description() + project.do(changes) + after_code = resource.read() + after = _run(after_code, directory) + evidence.update( + after_source=after_code, + after=after, + actual_source_change=after_code != code, + runtime_preserved=after == before, + ) + assert (directory / "sentinel.txt").read_bytes() == original["sentinel.txt"] + assert after_code != code + assert after == before, evidence + if selected == "parent_slot": + assert " renamed_callback = module_callback" in after_code + assert "Child().renamed_callback(Other())" in after_code + else: + assert "self.changed = " + value in after_code + expected_read = ( + "print(result.changed)" + if selected == expected_class + else "print(result.field)" + ) + assert expected_read in after_code + finally: + project.close() + (tmp_path / "evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", encoding="utf-8" + ) + + +@pytest.mark.parametrize("fixture", list(CASES)) +@pytest.mark.parametrize("field_role", ["primary", "Other"]) +def test_receiver_storage_regressions_preserve_execution(tmp_path, fixture, field_role): + code, expected_class, stdout = CASES[fixture] + primary = "Child" if fixture == "inherited" else "Owner" + selected = primary if field_role == "primary" else "Other" + _exercise(tmp_path, fixture, code, expected_class, stdout, selected) + + +def test_inherited_callback_slot_rename_preserves_source_identity(tmp_path): + _exercise( + tmp_path, "inherited_parent_slot", INHERITED, "Child", "3\n", "parent_slot" + ) + + +IMPORTED = ( + "from helper import module_callback\n" + + OWNER + + CLASS_ALIAS + + OTHER + + "result = Owner().callback(Other())\nprint(result.field)\n" +) + + +def _imported_project(directory): + project, resource = _project(directory, IMPORTED) + helper = project.root.create_file("helper.py") + helper.write(MODULE_FUNCTION) + return project, resource, helper + + +@pytest.mark.parametrize("selected", ["Owner", "Other", "class_slot", "function"]) +def test_imported_function_class_storage_preserves_execution(tmp_path, selected): + query_project, query_resource, query_helper = _imported_project(tmp_path / "query") + try: + scope = query_project.get_pymodule(query_resource).get_scope() + result = scope.lookup("result").get_object() + function = scope.lookup("module_callback").get_object() + helper_module = query_project.get_pymodule(query_helper) + query = { + "unknown": result is pyobjects.get_unknown(), + "type_expected": result.get_type() is scope.lookup("Owner").get_object(), + "function_is_canonical": function + is helper_module.get_scope().lookup("module_callback").get_object(), + "function_module_is_helper": function.get_module() is helper_module, + } + finally: + query_project.close() + directory = tmp_path / "cold" + project, resource, helper = _imported_project(directory) + original = {p.name: p.read_bytes() for p in directory.iterdir() if p.is_file()} + target, text = ( + (helper, MODULE_FUNCTION) if selected == "function" else (resource, IMPORTED) + ) + if selected == "function": + offset = text.index("module_callback") + 1 + new_name = "renamed_function" + elif selected == "class_slot": + offset = text.index(" callback =") + len(" ") + 1 + new_name = "renamed_callback" + else: + value = "1" if selected == "Owner" else "2" + offset = text.index("self.field = " + value) + len("self.") + 1 + new_name = "changed" + evidence = { + "fixture": "imported_function", + "selected": selected, + "query": query, + "offset": offset, + "target": target.path, + "new_name": new_name, + "query_project_separate_from_cold": True, + "before_source": {"subject.py": resource.read(), "helper.py": helper.read()}, + "original_bytes_hex": {name: data.hex() for name, data in original.items()}, + } + try: + before = _run(IMPORTED, directory) + evidence["before"] = before + assert before == (0, "1\n", "") + changes = Rename(project, target, offset).get_changes(new_name) + assert original == {name: (directory / name).read_bytes() for name in original} + evidence["changeset"] = changes.get_description() + project.do(changes) + after_source = {"subject.py": resource.read(), "helper.py": helper.read()} + after_bytes = {name: (directory / name).read_bytes() for name in original} + after = _run(resource.read(), directory) + evidence.update( + after_source=after_source, + after_bytes_hex={name: data.hex() for name, data in after_bytes.items()}, + after=after, + actual_source_change=after_bytes != original, + runtime_preserved=after == before, + ) + assert after_bytes["sentinel.txt"] == original["sentinel.txt"] + assert after_bytes != original + assert after == before, evidence + assert query["function_is_canonical"] and query["function_module_is_helper"] + subject = after_source["subject.py"] + if selected == "function": + assert "def renamed_function(" in after_source["helper.py"] + assert "from helper import renamed_function" in subject + assert "callback = renamed_function" in subject + elif selected == "class_slot": + assert "renamed_callback = module_callback" in subject + assert "Owner().renamed_callback(Other())" in subject + else: + assert "self.changed = " + value in subject + assert ( + "print(result.changed)" + if selected == "Owner" + else "print(result.field)" + ) in subject + finally: + project.close() + (tmp_path / "evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", encoding="utf-8" + ) diff --git a/ropetest/refactor/class_stored_function_test.py b/ropetest/refactor/class_stored_function_test.py new file mode 100644 index 000000000..04b56a4c9 --- /dev/null +++ b/ropetest/refactor/class_stored_function_test.py @@ -0,0 +1,356 @@ +import json +import os +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base import evaluate, pyobjects +from rope.base.project import Project +from rope.refactor.rename import Rename + +OWNER = dedent("""\ + class Owner: + def __init__(self): + self.field = 1 +""") +OTHER = dedent("""\ + class Other: + def __init__(self): + self.field = 2 +""") +MODULE_FUNCTION = dedent("""\ + def module_callback(self, item): + return self +""") +CLASS_ALIAS = " callback = module_callback\n" +PRINT_RESULT = "print(result.field)\n" + + +def _fixtures(): + stored = MODULE_FUNCTION + OWNER + CLASS_ALIAS + OTHER + return { + "direct_instance": ( + stored + "result = Owner().callback(Other())\n" + PRINT_RESULT, + "Owner", + "1\n", + ), + "saved_bound": ( + stored + + "saved = Owner().callback\nresult = saved(Other())\n" + + PRINT_RESULT, + "Owner", + "1\n", + ), + "class_unbound": ( + stored + "result = Owner.callback(Owner(), Other())\n" + PRINT_RESULT, + "Owner", + "1\n", + ), + "instance_own_function": ( + stored + + "instance = Owner()\ninstance.callback = module_callback\n" + + "result = instance.callback(Other(), Owner())\n" + + PRINT_RESULT, + "Other", + "2\n", + ), + "ordinary_method": ( + OWNER + + " def callback(self, item):\n return self\n" + + OTHER + + "result = Owner().callback(Other())\n" + + PRINT_RESULT, + "Owner", + "1\n", + ), + "lexical_global": ( + OTHER + + "GLOBAL = Other()\n" + + "def module_callback(self, item):\n return GLOBAL\n" + + OWNER + + CLASS_ALIAS + + "result = Owner().callback(Other())\n" + + PRINT_RESULT, + "Other", + "2\n", + ), + } + + +def _run(code, directory): + compile((directory / "subject.py").read_bytes(), "subject.py", "exec") + env = os.environ.copy() + env.pop("PYTHONPATH", None) + result = subprocess.run( + [sys.executable, "-B", "subject.py"], + cwd=directory, + env=env, + capture_output=True, + text=True, + timeout=45, + ) + return result.returncode, result.stdout, result.stderr + + +def _project(directory, code): + directory.mkdir() + (directory / "subject.py").write_text(code, encoding="utf-8") + (directory / "sentinel.txt").write_text("untouched\n", encoding="utf-8") + project = Project( + directory, + ropefolder=None, + save_history=False, + save_objectdb=False, + perform_doa=False, + automatic_soa=False, + ) + return project, project.get_file("subject.py") + + +def _query(directory, code, expected_class): + project, resource = _project(directory, code) + try: + module = project.get_pymodule(resource) + scope = module.get_scope() + expected_type = scope.lookup(expected_class).get_object() + other_type = scope.lookup( + "Other" if expected_class == "Owner" else "Owner" + ).get_object() + result = scope.lookup("result").get_object() + result_type = result.get_type() + finder = evaluate.ScopeNameFinder(module) + read = finder.get_pyname_at( + code.index("print(result.field)") + len("print(result.") + 1 + ) + expected_field = expected_type.get_attribute("field") + return { + "expected_class": expected_class, + "result_unknown": result is pyobjects.get_unknown(), + "result_type_expected": result_type is expected_type, + "result_type_other": result_type is other_type, + "field_is_expected_binding": read is expected_field, + "field_unknown": read is None + or read.get_object() is pyobjects.get_unknown(), + } + finally: + project.close() + + +@pytest.mark.parametrize("fixture_name", list(_fixtures())) +@pytest.mark.parametrize("selected", ["Owner", "Other"]) +def test_class_stored_function_field_rename_preserves_execution( + tmp_path, fixture_name, selected +): + code, expected_class, expected_stdout = _fixtures()[fixture_name] + query = _query(tmp_path / "separate-query", code, expected_class) + project, resource = _project(tmp_path / "cold-rename", code) + directory = tmp_path / "cold-rename" + original = { + path.name: path.read_bytes() for path in directory.iterdir() if path.is_file() + } + offset = ( + code.index("self.field = " + ("1" if selected == "Owner" else "2")) + + len("self.") + + 1 + ) + evidence = { + "fixture": fixture_name, + "selected": selected, + "query": query, + "before_source": code, + "offset": offset, + "new_name": "changed", + "cold_project_has_no_query_before_rename": True, + "original_file_bytes_hex": { + name: data.hex() for name, data in original.items() + }, + } + try: + before = _run(code, directory) + evidence["before"] = before + assert before == (0, expected_stdout, "") + changes = Rename(project, resource, offset).get_changes("changed") + assert original == {name: (directory / name).read_bytes() for name in original} + evidence["changeset_description"] = changes.get_description() + project.do(changes) + after_code = resource.read() + after = _run(after_code, directory) + evidence.update( + after_source=after_code, + after=after, + actual_source_change=after_code != code, + ) + assert (directory / "sentinel.txt").read_bytes() == original["sentinel.txt"] + assert after_code != code + changed_definition = "self.changed = " + ("1" if selected == "Owner" else "2") + assert changed_definition in after_code + evidence["runtime_preserved"] = after == before + assert after == before, evidence + expected_read = ( + "print(result.changed)" + if selected == expected_class + else "print(result.field)" + ) + assert expected_read in after_code + # A known wrong public value is an error; unknown alone is not a red case. + assert not query["result_type_other"], query + finally: + project.close() + (tmp_path / "evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", encoding="utf-8" + ) + + +@pytest.mark.parametrize("selected", ["class_slot", "module_function"]) +def test_class_stored_function_name_rename_preserves_identity(tmp_path, selected): + code, _, expected_stdout = _fixtures()["direct_instance"] + directory = tmp_path / "cold-rename" + project, resource = _project(directory, code) + original = { + path.name: path.read_bytes() for path in directory.iterdir() if path.is_file() + } + offset = ( + code.index(" callback =") + len(" ") + 1 + if selected == "class_slot" + else code.index("def module_callback") + len("def ") + 1 + ) + new_name = "renamed_callback" + evidence = { + "fixture": "canonical_name_rename", + "selected": selected, + "before_source": code, + "offset": offset, + "new_name": new_name, + "cold_project_has_no_query_before_rename": True, + "original_file_bytes_hex": { + name: data.hex() for name, data in original.items() + }, + } + try: + before = _run(code, directory) + evidence["before"] = before + assert before == (0, expected_stdout, "") + changes = Rename(project, resource, offset).get_changes(new_name) + assert original == {name: (directory / name).read_bytes() for name in original} + evidence["changeset_description"] = changes.get_description() + project.do(changes) + after_code = resource.read() + after = _run(after_code, directory) + evidence.update( + after_source=after_code, + after=after, + actual_source_change=after_code != code, + ) + assert (directory / "sentinel.txt").read_bytes() == original["sentinel.txt"] + assert after_code != code + assert after == before, evidence + if selected == "class_slot": + assert " renamed_callback = module_callback\n" in after_code + assert "Owner().renamed_callback(Other())" in after_code + assert "def module_callback(self, item):" in after_code + else: + assert "def renamed_callback(self, item):" in after_code + assert " callback = renamed_callback\n" in after_code + assert "Owner().callback(Other())" in after_code + evidence["runtime_preserved"] = True + finally: + project.close() + (tmp_path / "evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", encoding="utf-8" + ) + + +@pytest.mark.parametrize("query_order", [("first", "second"), ("second", "first")]) +@pytest.mark.parametrize("selected", ["Owner", "SecondOwner"]) +def test_shared_class_function_keeps_receivers_separate( + tmp_path, query_order, selected +): + code = ( + MODULE_FUNCTION + + OWNER + + CLASS_ALIAS + + "class SecondOwner:\n def __init__(self):\n self.field = 3\n" + + CLASS_ALIAS + + OTHER + + "first = Owner().callback(Other())\n" + + "second = SecondOwner().callback(Other())\n" + + "print(first.field, second.field)\n" + ) + query_project, query_resource = _project(tmp_path / "separate-query", code) + try: + scope = query_project.get_pymodule(query_resource).get_scope() + expected_types = { + "first": scope.lookup("Owner").get_object(), + "second": scope.lookup("SecondOwner").get_object(), + } + other_type = scope.lookup("Other").get_object() + queries = [] + for name in (*query_order, *query_order): + value = scope.lookup(name).get_object() + value_type = value.get_type() + queries.append( + { + "name": name, + "unknown": value is pyobjects.get_unknown(), + "type_expected": value_type is expected_types[name], + "type_other": value_type is other_type, + } + ) + finally: + query_project.close() + directory = tmp_path / "cold-rename" + project, resource = _project(directory, code) + original = { + path.name: path.read_bytes() for path in directory.iterdir() if path.is_file() + } + offset = ( + code.index("self.field = " + ("1" if selected == "Owner" else "3")) + + len("self.") + + 1 + ) + evidence = { + "fixture": "shared_two_classes-" + "-".join(query_order), + "selected": selected, + "queries": queries, + "before_source": code, + "offset": offset, + "new_name": "changed", + "cold_project_has_no_query_before_rename": True, + "original_file_bytes_hex": { + name: data.hex() for name, data in original.items() + }, + } + try: + before = _run(code, directory) + evidence["before"] = before + assert before == (0, "1 3\n", "") + changes = Rename(project, resource, offset).get_changes("changed") + assert original == {name: (directory / name).read_bytes() for name in original} + evidence["changeset_description"] = changes.get_description() + project.do(changes) + after_code = resource.read() + after = _run(after_code, directory) + evidence.update( + after_source=after_code, + after=after, + actual_source_change=after_code != code, + runtime_preserved=after == before, + ) + assert (directory / "sentinel.txt").read_bytes() == original["sentinel.txt"] + assert after_code != code + assert "self.changed = " + ("1" if selected == "Owner" else "3") in after_code + assert after == before, evidence + expected_read = ( + "print(first.changed, second.field)" + if selected == "Owner" + else "print(first.field, second.changed)" + ) + assert expected_read in after_code + assert not any(item["type_other"] for item in queries), queries + finally: + project.close() + (tmp_path / "evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", encoding="utf-8" + ) diff --git a/ropetest/refactor/class_transfer_test.py b/ropetest/refactor/class_transfer_test.py new file mode 100644 index 000000000..1accf0705 --- /dev/null +++ b/ropetest/refactor/class_transfer_test.py @@ -0,0 +1,190 @@ +import runpy + +import pytest + +from rope.base.exceptions import RefactoringError +from rope.base.project import Project +from rope.refactor.move import MoveMethod + + +def test_same_module_transfer_preserves_local_and_global_bindings(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("handler.py") + resource.write( + "import math\n" + "class Handler:\n" + " class Helpers:\n" + " @staticmethod\n" + " def calculate(value):\n" + " result = math.floor(value)\n" + " return result\n" + ) + source = resource.read() + changes = MoveMethod.get_class_changes( + project, + [(resource, source.index("calculate"), "calculate")], + resource, + source.index("Handler"), + resources=[resource], + ) + project.do(changes) + namespace = runpy.run_path(resource.real_path) + assert namespace["Handler"].calculate(3.5) == 3 + assert not hasattr(namespace["Handler"].Helpers, "calculate") + finally: + project.close() + + +@pytest.mark.parametrize("declaration", ["factor = 2\n", ""]) +def test_same_module_transfer_rejects_unproven_global_identity(tmp_path, declaration): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("handler.py") + resource.write( + declaration + "class Handler:\n" + " class Helpers:\n" + " @staticmethod\n" + " def calculate(value):\n" + " global factor\n" + " return value * factor\n" + ) + source = resource.read() + with pytest.raises( + RefactoringError, + match="(?:changes binding|Unproven method dependency): factor", + ): + MoveMethod.get_class_changes( + project, + [(resource, source.index("calculate"), "calculate")], + resource, + source.index("Handler"), + resources=[resource], + ) + assert resource.read() == source + finally: + project.close() + + +def test_cross_module_transfer_preserves_branch_local_source_binding(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + source = project.root.create_file("source.py") + destination = project.root.create_file("destination.py") + source.write( + "class Source:\n" + " @staticmethod\n" + " def choose(value):\n" + " if value:\n" + ' msg = f"value: {value}"\n' + " raise ValueError(msg)\n" + " return value\n" + ) + destination.write("class Target:\n pass\n") + before = runpy.run_path(source.real_path)["Source"] + assert before.choose(0) == 0 + with pytest.raises(ValueError, match="value: 3"): + before.choose(3) + changes = MoveMethod.get_class_changes( + project, + [(source, source.read().index("choose"), "choose")], + destination, + destination.read().index("Target"), + resources=[source, destination], + ) + project.do(changes) + after = runpy.run_path(destination.real_path)["Target"] + assert after.choose(0) == 0 + with pytest.raises(ValueError, match="value: 3"): + after.choose(3) + assert not hasattr(runpy.run_path(source.real_path)["Source"], "choose") + finally: + project.close() + + +def test_branch_value_overlay_does_not_admit_global_assignment(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + source = project.root.create_file("source.py") + destination = project.root.create_file("destination.py") + source.write( + "msg = 'original'\n" + "class Source:\n" + " @staticmethod\n" + " def choose(value):\n" + " global msg\n" + " if value:\n" + ' msg = f"value: {value}"\n' + " return msg\n" + ) + destination.write("class Target:\n pass\n") + before = source.read(), destination.read() + with pytest.raises(RefactoringError): + MoveMethod.get_class_changes( + project, + [(source, source.read().index("choose"), "choose")], + destination, + destination.read().index("Target"), + resources=[source, destination], + ) + assert (source.read(), destination.read()) == before + finally: + project.close() + + +def test_cross_module_transfer_rejects_mutation_through_builtins_namespace(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + origin = project.root.create_file("origin.py") + target = project.root.create_file("target.py") + source = ( + "class Origin:\n @staticmethod\n def compute():\n" + " return 9\n" + ) + destination = ( + "import builtins\noriginal = builtins.staticmethod\n" + "builtins.__dict__['staticmethod'] = lambda f: 42\n" + "class Target:\n pass\n" + "builtins.__dict__['staticmethod'] = original\n" + ) + origin.write(source) + target.write(destination) + with pytest.raises(RefactoringError, match="destination namespace bindings"): + MoveMethod.get_class_changes( + project, + [(origin, source.index("compute"), "compute")], + target, + destination.index("Target"), + resources=[origin, target], + ) + assert origin.read() == source + assert target.read() == destination + finally: + project.close() + + +def test_cross_module_transfer_preserves_builtin_type_annotations(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + origin = project.root.create_file("origin.py") + target = project.root.create_file("target.py") + origin.write( + "class Origin:\n @staticmethod\n" + " def compute(value: int) -> int:\n return value + 1\n" + ) + target.write("class Target:\n pass\n") + before = runpy.run_path(origin.real_path)["Origin"].compute + project.do( + MoveMethod.get_class_changes( + project, + [(origin, origin.read().index("compute"), "compute")], + target, + target.read().index("Target"), + resources=[origin, target], + ) + ) + after = runpy.run_path(target.real_path)["Target"].compute + assert after(4) == before(4) + assert after.__annotations__ == before.__annotations__ + finally: + project.close() diff --git a/ropetest/refactor/extracttest.py b/ropetest/refactor/extracttest.py index 53a225d6e..8fe3525f5 100644 --- a/ropetest/refactor/extracttest.py +++ b/ropetest/refactor/extracttest.py @@ -108,6 +108,28 @@ def extracted(): """) self.assertEqual(expected, refactored) + def test_extract_function_with_fstring(self): + code = dedent("""\ + def main(): + h = 1 + g = f"#{h}" + print(g) + """) + start, end = self._convert_line_range_to_offset(code, 3, 3) + refactored = self.do_extract_method(code, start, end, "extracted") + + expected = dedent("""\ + def main(): + h = 1 + g = extracted(h) + print(g) + + def extracted(h): + g = f"#{h}" + return g + """) + self.assertEqual(expected, refactored) + def test_extract_function_containing_dict_generalized_unpacking(self): code = dedent("""\ def a_func(dict1): @@ -1314,7 +1336,6 @@ def new_func(a_var): """) self.assertEqual(expected, refactored) - @unittest.expectedFailure @testutils.only_for_versions_higher("3.6") def test_extract_method_f_string_false_format_value_in_regular_string(self): code = dedent("""\ @@ -1788,6 +1809,153 @@ def new_func(): """) self.assertEqual(expected, refactored) + def test_extract_method_in_class_body(self): + code = dedent("""\ + class TSV: + delimiter = "\\t" + + print(TSV.delimiter) + """) + start = code.index('"\\t"') + end = start + len('"\\t"') + refactored = self.do_extract_method(code, start, end, "extracted") + expected = dedent("""\ + + def extracted(): + return "\\t" + + class TSV: + delimiter = extracted() + + print(TSV.delimiter) + """) + self.assertEqual(expected, refactored) + + def test_extract_method_in_class_body_reading_class_variables(self): + code = dedent("""\ + class A: + a = 1 + b = a + 2 + """) + start = code.index("a + 2") + end = start + len("a + 2") + refactored = self.do_extract_method(code, start, end, "extracted") + expected = dedent("""\ + + def extracted(a): + return a + 2 + + class A: + a = 1 + b = extracted(a) + """) + self.assertEqual(expected, refactored) + + def test_extract_method_in_decorated_class_body(self): + code = dedent("""\ + @decorator + class A: + a = 1 + 2 + """) + start = code.index("1 + 2") + end = start + len("1 + 2") + refactored = self.do_extract_method(code, start, end, "extracted") + expected = dedent("""\ + + def extracted(): + return 1 + 2 + + @decorator + class A: + a = extracted() + """) + self.assertEqual(expected, refactored) + + def test_extract_method_in_class_body_inside_function(self): + code = dedent("""\ + def f(): + class A: + a = 1 + 2 + return A + """) + start = code.index("1 + 2") + end = start + len("1 + 2") + refactored = self.do_extract_method(code, start, end, "extracted") + expected = dedent("""\ + def f(): + + def extracted(): + return 1 + 2 + + class A: + a = extracted() + return A + """) + self.assertEqual(expected, refactored) + + def test_extract_method_in_nested_class_body(self): + code = dedent("""\ + class A: + class B: + a = 1 + 2 + """) + start = code.index("1 + 2") + end = start + len("1 + 2") + refactored = self.do_extract_method(code, start, end, "extracted") + expected = dedent("""\ + + def extracted(): + return 1 + 2 + + class A: + class B: + a = extracted() + """) + self.assertEqual(expected, refactored) + + def test_global_extract_method_in_class_body(self): + code = dedent("""\ + def f(): + class A: + a = 1 + 2 + return A + """) + start = code.index("1 + 2") + end = start + len("1 + 2") + refactored = self.do_extract_method( + code, start, end, "extracted", global_=True + ) + expected = dedent("""\ + def f(): + class A: + a = extracted() + return A + + def extracted(): + return 1 + 2 + """) + self.assertEqual(expected, refactored) + + def test_global_extract_method_in_module_level_class_body(self): + code = dedent("""\ + class A: + a = 1 + 2 + """) + start = code.index("1 + 2") + end = start + len("1 + 2") + refactored = self.do_extract_method( + code, start, end, "extracted", global_=True + ) + expected = dedent("""\ + + def extracted(): + return 1 + 2 + + class A: + a = extracted() + """) + self.assertEqual(expected, refactored) + def test_where_to_search_when_extracting_global_names(self): code = dedent("""\ def a(): @@ -2403,18 +2571,11 @@ def foo(a): start, end = code.index(extract_target), code.index(extract_target) + len( extract_target ) - refactored = self.do_extract_method(code, start, end, "new_func") - expected = dedent("""\ - def foo(a): - if i, c := new_func(a): - i += 1 - c += 1 - print(i) - - def new_func(a): - return (i := a == (c := 5)) - """) - self.assertEqual(expected, refactored) + with self.assertRaisesRegex( + rope.base.exceptions.RefactoringError, + "Extracted piece cannot contain named expression \\(:= operator\\).", + ): + self.do_extract_method(code, start, end, "new_func") @testutils.only_for_versions_higher("3.8") def test_extract_function_expression_with_inline_assignment_in_inner_expression( @@ -2542,27 +2703,6 @@ def new_func(x): """) self.assertEqual(expected, refactored) - @testutils.only_for_versions_higher("3.5") - @testutils.only_for_versions_lower("3.8") - def test_extract_refactor_containing_async_for_loop_should_error_before_py38(self): - """ - Refactoring async/await syntaxes is only supported in Python 3.8 and - higher because support for ast.PyCF_ALLOW_TOP_LEVEL_AWAIT was only - added to the standard library in Python 3.8. - """ - code = dedent("""\ - async def my_func(my_list): - async for x in my_list: - var = x + 1 - return var - """) - start, end = self._convert_line_range_to_offset(code, 2, 3) - with self.assertRaisesRegex( - rope.base.exceptions.RefactoringError, - "Extracted piece can only have async/await statements if Rope is running on Python 3.8 or higher", - ): - self.do_extract_method(code, start, end, "new_func") - @testutils.only_for_versions_higher("3.8") def test_extract_refactor_containing_async_for_loop_is_supported_after_py38(self): code = dedent("""\ diff --git a/ropetest/refactor/importutilstest.py b/ropetest/refactor/importutilstest.py index a2ebd49d1..1d7a444ac 100644 --- a/ropetest/refactor/importutilstest.py +++ b/ropetest/refactor/importutilstest.py @@ -1,6 +1,8 @@ import unittest from textwrap import dedent +import pytest + from rope.base.prefs import ( DEFAULT_IMPORT_STYLE, ImportPrefs, @@ -8,7 +10,12 @@ Prefs, get_preferred_import_style, ) -from rope.refactor.importutils import ImportTools, add_import, importinfo +from rope.refactor.importutils import ( + ImportTools, + add_import, + get_module_imports, + importinfo, +) from ropetest import testutils @@ -1707,6 +1714,23 @@ def test_organizing_imports_all_star(self): pymod = self.project.get_pymodule(self.mod) self.assertEqual(expected, self.import_tools.organize_imports(pymod)) + def test_package_used_imports_include_its_declared_exports(self): + self.pkg1.get_child("__init__.py").write(dedent("""\ + from os import path + + + __all__ = ["path"] + """)) + package = self.project.get_module("pkg1") + module_imports = get_module_imports(self.project, package) + self.assertEqual( + ["from os import path"], + [ + statement.get_import_statement() + for statement in module_imports.get_used_imports(package) + ], + ) + def test_organizing_imports_all_star_with_variables(self): code = expected = dedent("""\ from package import name_one, name_two @@ -1812,7 +1836,7 @@ def test_organizing_imports_all_star_starjoin(self): pymod = self.project.get_pymodule(self.mod) self.assertEqual(expected, self.import_tools.organize_imports(pymod)) - @testutils.time_limit(60) + @pytest.mark.timeout(60) def test_organizing_imports_all_star_no_infinite_loop(self): code = expected = dedent("""\ from package import name_one, name_two @@ -1948,6 +1972,37 @@ def import_filter(import_stmt): return import_filter + def test_filtered_duplicate_removal_preserves_distinct_from_imports(self): + for imported in ("mod3", "mod3 as other", "*"): + with self.subTest(imported=imported): + source = dedent(f"""\ + from pkg2 import mod2 + from pkg2 import {imported} + """) + self.mod.write(source) + pymod = self.project.get_pymodule(self.mod) + module_imports = self.import_tools.module_imports( + pymod, self._line_filter(2) + ) + module_imports.remove_duplicates() + self.assertEqual(source, module_imports.get_changed_source()) + + def test_filtered_duplicate_removal_still_removes_covered_imports(self): + for imported in ("mod2", "mod2, mod3", "*"): + with self.subTest(imported=imported): + expected = dedent(f"""\ + from pkg2 import {imported} + """) + self.mod.write(expected + dedent("""\ + from pkg2 import mod2 + """)) + pymod = self.project.get_pymodule(self.mod) + module_imports = self.import_tools.module_imports( + pymod, self._line_filter(2) + ) + module_imports.remove_duplicates() + self.assertEqual(expected, module_imports.get_changed_source()) + def test_filtered_expand_stars(self): self.pkg1.get_child("__init__.py").write("var1 = 1\n") self.pkg2.get_child("__init__.py").write("var2 = 1\n") diff --git a/ropetest/refactor/inline_annotations_test.py b/ropetest/refactor/inline_annotations_test.py new file mode 100644 index 000000000..3e4953c0d --- /dev/null +++ b/ropetest/refactor/inline_annotations_test.py @@ -0,0 +1,424 @@ +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base.exceptions import RefactoringError +from rope.base.project import Project +from rope.refactor.inline import create_inline + + +@pytest.fixture +def project(tmp_path): + project = Project( + str(tmp_path), save_objectdb=False, save_history=False, automatic_soa=False + ) + yield project + project.close() + + +def module(project, name, source): + resource = project.root.create_file(name + ".py") + resource.write(dedent(source)) + return resource + + +def execute(project, resource): + result = subprocess.run( + [sys.executable, str(resource.real_path)], + cwd=project.address, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +@pytest.fixture(params=[True, False], ids=["future", "default"]) +def deferred_prefix(request): + if not request.param and sys.version_info < (3, 14): + pytest.skip("default annotations are deferred on Python 3.14+") + return "from __future__ import annotations\n" if request.param else "" + + +@pytest.mark.parametrize("remove", [True, False]) +def test_refuses_changing_annotation_name_binding(project, deferred_prefix, remove): + source = module(project, "source", deferred_prefix + dedent('''\ + original = int + target = original + def func(x: target): + pass + original = str + import typing + print(typing.get_type_hints(func)["x"] is int) + ''')) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes(remove=remove) + assert source.read() == before + assert execute(project, source) == output + + +def test_refuses_moving_side_effect_into_annotation(project, deferred_prefix): + source = module(project, "source", deferred_prefix + dedent('''\ + events = [] + def make(): + events.append("called") + return int + target = make() + def func() -> target: + pass + print(events) + import typing + print(typing.get_type_hints(func)["return"] is int) + print(events) + ''')) + before = source.read() + output = execute(project, source) + assert output == "['called']\nTrue\n['called']\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.parametrize("definition,owner,key", [ + (dedent("""\ + async def func(x: target): + pass + """), "func", "x"), + (dedent("""\ + def func(x: target, /): + pass + """), "func", "x"), + (dedent("""\ + def func(*, x: target): + pass + """), "func", "x"), + (dedent("""\ + def func(*x: target): + pass + """), "func", "x"), + (dedent("""\ + def func(**x: target): + pass + """), "func", "x"), + (dedent("""\ + def func() -> target: + pass + """), "func", "return"), + (dedent("""\ + class Owner: + x: target + """), "Owner", "x"), + (dedent("""\ + x: target + import sys + """), "sys.modules[__name__]", "x"), + (dedent("""\ + def 函数(x: ( + list[target] + )): + pass + """), "函数", "x"), +]) +def test_refuses_annotation_kinds(project, deferred_prefix, definition, owner, key): + comparison = "list[int]" if "list[target]" in definition else "int" + code = deferred_prefix + "original = int\ntarget = original\n" + definition + code += "original = str\nimport typing\n" + code += f'print(typing.get_type_hints({owner})["{key}"] == {comparison})\n' + source = module(project, "source", code) + assert execute(project, source) == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, code.index("target") + 1).get_changes() + assert source.read() == code + assert execute(project, source) == "True\n" + + +@pytest.mark.parametrize("qualified", [True, False]) +def test_cross_module_annotation_is_refused(project, deferred_prefix, qualified): + source = module(project, "source", "original = int\ntarget = original\n") + imported = "source.target" if qualified else "target" + import_line = "import source\n" if qualified else "from source import target\n" + code = deferred_prefix + import_line + code += f"def func(x: {imported}):\n pass\n" + code += 'import typing\nprint(typing.get_type_hints(func)["x"] is int)\n' + entry = module(project, "entry", code) + before = {source: source.read(), entry: entry.read()} + assert execute(project, entry) == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, entry, code.index("target", code.index("def func")) + 1).get_changes(only_current=True, remove=False) + assert {r: r.read() for r in before} == before + assert execute(project, entry) == "True\n" + + +@pytest.mark.parametrize("remove", [False, True]) +def test_only_current_eager_reference(project, deferred_prefix, remove): + source = module(project, "source", deferred_prefix + dedent('''\ + target = int + def func(x: target): + pass + result = target + import typing + print(result is typing.get_type_hints(func)["x"]) + ''')) + before = source.read() + output = execute(project, source) + inline = create_inline(project, source, before.rindex("target") + 1) + if remove: + with pytest.raises(RefactoringError, match="deferred annotation"): + inline.get_changes(only_current=True) + assert source.read() == before + else: + project.do(inline.get_changes(only_current=True, remove=False)) + assert "result = int" in source.read() + assert "target = int" in source.read() + assert execute(project, source) == output == "True\n" + + +def test_eager_annotations_allow_inline(project): + if sys.version_info >= (3, 14): + pytest.skip("default annotations are deferred on Python 3.14+") + source = module(project, "source", '''\ + original = int + target = original + def func(x: target): + pass + original = str + import typing + print(typing.get_type_hints(func)["x"] is int) + ''') + before = execute(project, source) + project.do(create_inline(project, source, source.read().index("target") + 1).get_changes()) + assert "target" not in source.read() + assert execute(project, source) == before == "True\n" + + +def test_function_default_is_eager(project, deferred_prefix): + source = module(project, "source", deferred_prefix + dedent('''\ + original = int + target = original + def func(x: str = target): + return x + original = str + print(func() is int) + ''')) + before = execute(project, source) + project.do(create_inline(project, source, source.read().index("target") + 1).get_changes()) + assert "x: str = original" in source.read() + assert execute(project, source) == before == "True\n" + + +def test_shadowed_function_annotation_checks_runtime_namespace(project, deferred_prefix): + source = module(project, "source", deferred_prefix + dedent('''\ + target = int + def func(target: str): + def inner(x: target): + pass + return inner + result = target + import typing + print(result is int, typing.get_type_hints(func(str))["x"] is int) + ''')) + before = execute(project, source) + inline = create_inline(project, source, source.read().index("target") + 1) + if deferred_prefix: + code = source.read() + with pytest.raises(RefactoringError, match="deferred annotation"): + inline.get_changes() + assert source.read() == code + assert execute(project, source) == before == "True True\n" + else: + project.do(inline.get_changes()) + assert "result = int" in source.read() + assert "x: target" in source.read() + assert execute(project, source) == before == "True False\n" + + +def test_local_variable_annotation_is_not_evaluated(project, deferred_prefix): + source = module(project, "source", deferred_prefix + dedent('''\ + target = int + def func(): + x: target = 1 + return x + print(func()) + ''')) + before = execute(project, source) + project.do(create_inline(project, source, source.read().index("target") + 1).get_changes()) + assert "target = int" not in source.read() + assert execute(project, source) == before == "1\n" + + +@pytest.mark.parametrize("annotation", ["holder.x: target = 1", "holder[0]: target = 1", "(result): target = 1"]) +def test_non_simple_annotation_is_not_deferred(project, deferred_prefix, annotation): + if annotation.startswith("(result)"): + setup, access = "", "result" + elif "[0]" in annotation: + setup, access = "holder = [0]\n", "holder[0]" + else: + setup, access = "class Holder: pass\nholder = Holder()\n", "holder.x" + source = module(project, "source", deferred_prefix + "target = int\n" + setup + annotation + f"\nprint({access})\n") + before = execute(project, source) + project.do(create_inline(project, source, source.read().index("target") + 1).get_changes()) + assert "target = int" not in source.read() + assert execute(project, source) == before == "1\n" + + +def test_nested_eager_function_annotation_allows_inline(project): + if sys.version_info >= (3, 14): + pytest.skip("default annotations are deferred on Python 3.14+") + source = module(project, "source", '''\ + def outer(): + original = int + target = original + def inner(x: target): + pass + original = str + return inner + import typing + print(typing.get_type_hints(outer())["x"] is int) + ''') + before = execute(project, source) + project.do(create_inline(project, source, source.read().index("target") + 1).get_changes()) + assert "target" not in source.read() + assert execute(project, source) == before == "True\n" + + +def test_class_annotation_within_function_is_still_deferred(project, deferred_prefix): + source = module(project, "source", deferred_prefix + dedent('''\ + original = int + target = original + def factory(): + class Owner: + x: target + return Owner + original = str + import typing + print(typing.get_type_hints(factory())["x"] is int) + ''')) + before = source.read() + assert execute(project, source) == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + + +def test_class_shadowed_annotation_checks_runtime_namespace(project, deferred_prefix): + source = module(project, "source", deferred_prefix + dedent('''\ + target = int + class Owner: + target = str + x: target + result = target + import typing + print(result is int, typing.get_type_hints(Owner)["x"] is str) + ''')) + before = execute(project, source) + inline = create_inline(project, source, source.read().index("target") + 1) + if deferred_prefix: + code = source.read() + with pytest.raises(RefactoringError, match="deferred annotation"): + inline.get_changes() + assert source.read() == code + assert execute(project, source) == before == "True False\n" + else: + project.do(inline.get_changes()) + assert "result = int" in source.read() + assert "target = str" in source.read() + assert execute(project, source) == before == "True True\n" + + +def test_resources_sequence_is_preserved(project, deferred_prefix): + source = module(project, "source", deferred_prefix + "target = int\nresult = target\nprint(result is int)\n") + before = execute(project, source) + resources = (source,) + project.do(create_inline(project, source, source.read().index("target") + 1).get_changes(resources=resources)) + assert "target" not in source.read() + assert execute(project, source) == before == "True\n" + + +def test_function_body_shadowing_still_allows_inline(project, deferred_prefix): + source = module(project, "source", deferred_prefix + dedent('''\ + target = int + def func(target: str): + return target + result = target + print(result is int, func(str) is str) + ''')) + before = execute(project, source) + project.do(create_inline(project, source, source.read().index("target") + 1).get_changes()) + assert "result = int" in source.read() + assert "return target" in source.read() + assert execute(project, source) == before == "True True\n" + + +@pytest.mark.parametrize("remove", [False, True]) +@pytest.mark.parametrize("only_current", [False, True]) +def test_only_current_checks_annotation_in_third_module(project, deferred_prefix, remove, only_current): + source = module(project, "source", "target = int\n") + annotated = module(project, "annotated", deferred_prefix + "import source\ndef func(x: source.target):\n pass\n") + entry = module(project, "entry", '''\ + import source + import annotated + result = source.target + import typing + print(result is typing.get_type_hints(annotated.func)["x"]) + ''') + before = {r: r.read() for r in (source, annotated, entry)} + output = execute(project, entry) + inline = create_inline(project, entry, entry.read().rindex("target") + 1) + if remove: + with pytest.raises(RefactoringError, match="deferred annotation"): + inline.get_changes(only_current=only_current, resources=[entry, source]) + assert {r: r.read() for r in before} == before + else: + project.do(inline.get_changes(only_current=only_current, remove=False, resources=[entry, source])) + assert "result = int" in entry.read() + assert source.read() == before[source] + assert annotated.read() == before[annotated] + assert execute(project, entry) == output == "True\n" + + +def test_future_import_alias_preserves_runtime_global_binding(project): + source = module(project, "source", "target = int\n") + entry = module(project, "entry", '''\ + from __future__ import annotations + from source import target as Alias + class Owner: + Alias = str + x: Alias + import typing + print(typing.get_type_hints(Owner)["x"] is int) + ''') + before = {source: source.read(), entry: entry.read()} + assert execute(project, entry) == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, source.read().index("target") + 1).get_changes() + assert {r: r.read() for r in before} == before + assert execute(project, entry) == "True\n" + + +def test_future_qualified_annotation_preserves_runtime_global_binding(project): + source = module(project, "source", "target = int\n") + entry = module(project, "entry", '''\ + from __future__ import annotations + import source + class Holder: + target = str + class Owner: + source = Holder + x: source.target + result = source.target + import typing + print(result is typing.get_type_hints(Owner)["x"]) + ''') + before = {source: source.read(), entry: entry.read()} + assert execute(project, entry) == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, source.read().index("target") + 1).get_changes() + assert {r: r.read() for r in before} == before + assert execute(project, entry) == "True\n" diff --git a/ropetest/refactor/inline_lambda_test.py b/ropetest/refactor/inline_lambda_test.py new file mode 100644 index 000000000..16d87dc07 --- /dev/null +++ b/ropetest/refactor/inline_lambda_test.py @@ -0,0 +1,608 @@ +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base.exceptions import RefactoringError +from rope.base.project import Project +from rope.refactor.inline import InlineParameter, InlineVariable, create_inline + + +@pytest.fixture +def project(tmp_path): + project = Project( + str(tmp_path), save_objectdb=False, save_history=False, automatic_soa=False + ) + yield project + project.close() + + +def module(project, name, code): + resource = project.root.create_file(name + ".py") + resource.write(dedent(code)) + return resource + + +def execute(project, resource): + result = subprocess.run( + [sys.executable, resource.real_path], + cwd=project.address, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +SHADOW = dedent("""\ + target = 42 + value = (lambda target: target)(int) + print(value is int) +""") +NAMED_EXPR = dedent("""\ + target = 42 + value = (lambda: ((target := int), target)[1])() + print(value is int) + print(target) +""") + + +@pytest.mark.parametrize( + "code,expected", + [ + (SHADOW, "True\n"), + ( + dedent("""\ + def outer(): + target = 42 + return (lambda target: target)(int) + print(outer() is int) + """), + "True\n", + ), + ( + dedent("""\ + target = 42 + def func(value=(lambda target: target)(int)): + return value + print(func() is int) + """), + "True\n", + ), + ( + dedent("""\ + target = 42 + f = lambda target=target: target + print(f(), f(int) is int) + """), + "42 True\n", + ), + ( + dedent("""\ + target = 42 + value = (lambda target: (lambda value=target: value)())(int) + print(value is int) + """), + "True\n", + ), + (NAMED_EXPR, "True\n42\n"), + ], +) +def test_inlining_outer_variable_preserves_lambda_local_bindings( + project, code, expected +): + # Source bindings (#900) keep the lambda parameter distinct from the + # outer variable, so inlining the outer definition no longer needs to + # refuse: it substitutes only the real references and keeps the lambda + # binding intact, verified by execution before and after. + source = module(project, "source", code) + before = source.read() + assert execute(project, source) == expected + project.do( + create_inline(project, source, before.index("target =") + 1).get_changes() + ) + assert source.read() != before + assert "target = 42" not in source.read() + assert execute(project, source) == expected + + +def test_refuses_discarding_side_effectful_initializer_without_reads(project): + # The lambda parameter is not a read of the outer variable, so no + # reference survives; discarding ``target = make_target()`` would drop + # its side effect and change the output. + source = module( + project, + "source", + dedent("""\ + def make_target(): + print("initialized") + return 42 + target = make_target() + value = (lambda target: target)(int) + print(value is int) + """), + ) + before = source.read() + assert execute(project, source) == "initialized\nTrue\n" + with pytest.raises(RefactoringError, match="unused variable initializer"): + create_inline(project, source, before.index("target =") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == "initialized\nTrue\n" + + +@pytest.mark.parametrize( + "expression", + [ + "(lambda target, /: target)(int)", + "(lambda *, target: target)(target=int)", + "(lambda *target: target[0])(int)", + "(lambda **target: target['item'])(item=int)", + ], +) +def test_covers_parameter_kinds(project, expression): + source = module( + project, + "source", + dedent(f"""\ + target = 42 + value = {expression} + print(value is int) + """), + ) + before = source.read() + assert execute(project, source) == "True\n" + project.do( + create_inline(project, source, before.index("target") + 1).get_changes() + ) + assert "target = 42" not in source.read() + assert expression in source.read() + assert execute(project, source) == "True\n" + + +@pytest.mark.parametrize("constructor", [create_inline, InlineVariable]) +@pytest.mark.parametrize("remove", [True, False]) +@pytest.mark.parametrize( + "code,fragment,expected", + [ + (SHADOW, "target: target", "True\n"), + (SHADOW, "target)(int)", "True\n"), + (NAMED_EXPR, "target :=", "True\n42\n"), + (NAMED_EXPR, "target)[1]", "True\n42\n"), + ], +) +def test_refuses_selecting_lambda_local_binding( + project, constructor, remove, code, fragment, expected +): + source = module(project, "source", code) + before = source.read() + assert execute(project, source) == expected + with pytest.raises(RefactoringError, match="lambda-local"): + constructor(project, source, before.index(fragment) + 1).get_changes( + only_current=True, remove=remove + ) + assert source.read() == before + assert execute(project, source) == expected + + +@pytest.mark.parametrize("constructor", [create_inline, InlineParameter]) +@pytest.mark.parametrize("fragment", ["target: target", "target)(int)"]) +def test_does_not_mistake_lambda_binding_for_outer_parameter( + project, constructor, fragment +): + source = module( + project, + "source", + dedent("""\ + def outer(target=42): + return (lambda target: target)(int) + print(outer() is int) + """), + ) + before = source.read() + assert execute(project, source) == "True\n" + with pytest.raises(RefactoringError, match="lambda-local"): + constructor(project, source, before.index(fragment) + 1).get_changes() + assert source.read() == before + assert execute(project, source) == "True\n" + + +def test_other_module_lambda_shadowing_inline_cleans_up_import(project): + # The lambda parameter in the caller is not a false occurrence anymore, + # so inlining succeeds, discards the definition and removes the import + # the caller no longer needs. + source = module(project, "source", "target = 42\n") + caller = module( + project, + "caller", + dedent("""\ + from source import target + value = (lambda target: target)(int) + print(value is int) + """), + ) + assert execute(project, caller) == "True\n" + project.do(create_inline(project, source, 1).get_changes()) + assert source.read() == "" + assert "import" not in caller.read() + assert "lambda target: target" in caller.read() + assert execute(project, caller) == "True\n" + + +@pytest.mark.parametrize( + "code,fragment,expected", + [ + ( + SHADOW.replace("value =", "value = ('中文',").replace("(int)", "(int))[1]"), + "target)(int)", + "True\n", + ), + (SHADOW.replace("target", "目标"), "目标)(int)", "True\n"), + ( + dedent("""\ + target = 42 + class Box: + value = int + value = (lambda target: target.value)(Box) + print(value is int) + """), + "target.value", + "True\n", + ), + ], +) +def test_local_selection_uses_name_ranges(project, code, fragment, expected): + source = module(project, "source", code) + before = source.read() + assert execute(project, source) == expected + with pytest.raises(RefactoringError, match="lambda-local"): + create_inline(project, source, before.index(fragment) + 1).get_changes( + only_current=True, remove=False + ) + assert source.read() == before + assert execute(project, source) == expected + + +@pytest.mark.parametrize("remove", [True, False]) +def test_partial_eager_reference_does_not_touch_lambda_binding(project, remove): + source = module(project, "source", SHADOW + "print(target)\n") + before = source.read() + assert execute(project, source) == "True\n42\n" + project.do( + create_inline(project, source, before.rindex("target") + 1).get_changes( + only_current=True, remove=remove + ) + ) + assert "lambda target: target" in source.read() + assert execute(project, source) == "True\n42\n" + + +@pytest.mark.parametrize( + "code,fragment,expected,partial", + [ + ( + dedent("""\ + target = 42 + value = (lambda: target)() + print(value) + """), + "target =", + "42\n", + False, + ), + ( + dedent("""\ + target = 42 + value = (lambda other: (lambda value=target: value)())(0) + print(value) + """), + "target =", + "42\n", + False, + ), + ( + dedent("""\ + target = 42 + f = lambda target=target: target + print(f(), f(int) is int) + """), + "target: target", + "42 True\n", + True, + ), + ( + dedent("""\ + target = int + def consume(**kwargs): + return kwargs['target'] + value = (lambda other: consume(target=target))(0) + print(value is int) + """), + "target =", + "True\n", + False, + ), + ( + dedent("""\ + class Box: + target = int + pass + value = (lambda target: Box.target)(str) + print(value is int) + """), + "target =", + "True\n", + False, + ), + ( + dedent("""\ + target = 42 + value = (lambda: ((lambda: (target := int))(), target)[1])() + print(value) + """), + "target)[1]", + "42\n", + True, + ), + ( + dedent("""\ + target = 42 + value = (lambda: (target := int))() + print(value is int) + """), + "target =", + "True\n", + False, + ), + ], +) +def test_preserves_real_external_references_and_independent_names( + project, code, fragment, expected, partial +): + source = module(project, "source", code) + before = source.read() + assert execute(project, source) == expected + project.do( + create_inline(project, source, before.index(fragment) + 1).get_changes( + only_current=partial + ) + ) + assert source.read() != before + assert execute(project, source) == expected + + +def test_inline_discards_unused_outer_initializer_through_lambda_receiver(project): + # ``target.value`` resolves through the lambda-local receiver to Inner, + # so Outer.value has no surviving reads and its initializer is simply + # discarded; the lambda body is left untouched. + source = module( + project, + "source", + dedent("""\ + class Outer: + value = 42 + pass + class Inner: + value = int + target = Outer() + result = (lambda target: target.value)(Inner()) + print(result is int) + """), + ) + before = source.read() + assert execute(project, source) == "True\n" + project.do( + create_inline(project, source, before.index("value = 42") + 1).get_changes() + ) + assert "value = 42" not in source.read() + assert "(lambda target: target.value)(Inner())" in source.read() + assert execute(project, source) == "True\n" + + +def test_refuses_attribute_lookup_through_lambda_local_receiver(project): + # The receiver of ``.value`` is the value a lambda-local binding + # produces, so replacing the attribute there is not safe. + source = module( + project, + "source", + dedent("""\ + class Outer: + value = 42 + pass + class Inner: + value = int + target = Outer() + result = (lambda target: target)(Inner()).value + print(result is int) + """), + ) + before = source.read() + assert execute(project, source) == "True\n" + with pytest.raises(RefactoringError, match="lambda-local"): + create_inline(project, source, before.index("value = 42") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == "True\n" + + +def test_refuses_selecting_attribute_of_lambda_local_receiver(project): + source = module( + project, + "source", + dedent("""\ + class Outer: + value = 42 + pass + class Inner: + value = int + target = Outer() + result = (lambda target: target.value)(Inner()) + print(result is int) + """), + ) + before = source.read() + assert execute(project, source) == "True\n" + offset = before.index("target.value") + len("target.") + 1 + with pytest.raises(RefactoringError, match="lambda-local"): + create_inline(project, source, offset).get_changes(only_current=True) + assert source.read() == before + assert execute(project, source) == "True\n" + + +@pytest.mark.parametrize( + "expression", + ["(lambda target: Box)(str).value", "make_box(lambda target=target: target).value"], +) +def test_attribute_receiver_ignores_unrelated_lambda_bindings(project, expression): + source = module( + project, + "source", + dedent(f"""\ + target = 42 + class Box: + value = int + pass + def make_box(callback): + return Box() + result = {expression} + print(result is int) + print(target) + """), + ) + before = source.read() + assert execute(project, source) == "True\n42\n" + project.do( + create_inline(project, source, before.index("value = int") + 1).get_changes() + ) + assert source.read() != before + assert execute(project, source) == "True\n42\n" + + +@pytest.mark.parametrize( + "expression", + [ + "[item for item in target]", + "{item for item in target}", + "{item: item for item in target}", + "(item for item in target)", + ], +) +def test_comprehension_first_iterable_keeps_lambda_binding(project, expression): + source = module( + project, + "source", + dedent(f"""\ + target = [42] + value = (lambda target: {expression})([int]) + print(next(iter(value)) is int) + """), + ) + before = source.read() + assert execute(project, source) == "True\n" + offset = before.index("in target") + len("in ") + 1 + with pytest.raises(RefactoringError, match="lambda-local"): + create_inline(project, source, offset).get_changes( + only_current=True, remove=False + ) + assert source.read() == before + assert execute(project, source) == "True\n" + + +@pytest.mark.parametrize("fragment", ["in target", "if target"]) +def test_comprehension_later_iterable_and_condition_keep_lambda_binding( + project, fragment +): + source = module( + project, + "source", + dedent("""\ + target = [42] + value = (lambda target: [item for unused in [0] + for item in target if target[0] is int])([int]) + print(value == [int]) + """), + ) + before = source.read() + assert execute(project, source) == "True\n" + offset = before.index(fragment) + len("in ") + 1 + with pytest.raises(RefactoringError, match="lambda-local"): + create_inline(project, source, offset).get_changes( + only_current=True, remove=False + ) + assert source.read() == before + assert execute(project, source) == "True\n" + + +@pytest.mark.parametrize( + "code,fragment,expected", + [ + ( + dedent("""\ + class Box: + pass + target = Box() + receiver = Box() + value = (lambda target: [int for target.attr in [int]][0])(receiver) + print(value is int, receiver.attr is int, hasattr(target, "attr")) + """), + "for target.attr", + "True True False\n", + ), + ( + dedent("""\ + target = [str] + receiver = [str] + value = (lambda target: [int for target[0] in [int]][0])(receiver) + print(value is int, receiver[0] is int, target[0] is str) + """), + "for target[0]", + "True True True\n", + ), + ], +) +def test_comprehension_assignment_receiver_keeps_lambda_binding( + project, code, fragment, expected +): + source = module(project, "source", code) + before = source.read() + assert execute(project, source) == expected + offset = before.index(fragment) + len("for ") + 1 + with pytest.raises(RefactoringError, match="lambda-local"): + create_inline(project, source, offset).get_changes( + only_current=True, remove=False + ) + assert source.read() == before + assert execute(project, source) == expected + + +@pytest.mark.parametrize( + "expression", + [ + "[target for target in [int]]", + "[target for (target, unused) in [(int, 0)]]", + "[(lambda: target)() for target in [int]]", + ], +) +def test_comprehension_store_and_capture_allow_eager_reference(project, expression): + source = module( + project, + "source", + dedent(f"""\ + target = 42 + value = (lambda target: {expression})(str) + print(value[0] is int) + print(target) + """), + ) + before = source.read() + assert execute(project, source) == "True\n42\n" + offset = before.index("print(target)") + len("print(") + 1 + project.do( + create_inline(project, source, offset).get_changes( + only_current=True, remove=True + ) + ) + assert source.read() != before + assert "target = 42" not in source.read() + assert expression in source.read() + assert execute(project, source) == "True\n42\n" diff --git a/ropetest/refactor/inline_type_parameters_test.py b/ropetest/refactor/inline_type_parameters_test.py new file mode 100644 index 000000000..563b3bf04 --- /dev/null +++ b/ropetest/refactor/inline_type_parameters_test.py @@ -0,0 +1,690 @@ +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base.exceptions import RefactoringError +from rope.base.project import Project +from rope.refactor.inline import create_inline + +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 12), reason="type parameter syntax requires Python 3.12+" +) + + +@pytest.fixture +def project(tmp_path): + project = Project( + str(tmp_path), save_objectdb=False, save_history=False, automatic_soa=False + ) + yield project + project.close() + + +def module(project, name, code): + resource = project.root.create_file(name + ".py") + resource.write(dedent(code)) + return resource + + +def execute(project, resource): + result = subprocess.run( + [sys.executable, resource.real_path], + cwd=project.address, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +@pytest.mark.parametrize( + "declaration,owner", + [ + ("def func[T: target]():", "func"), + ("async def func[T: target]():", "func"), + ("class Owner[T: target]:", "Owner"), + ("def func[T: (target, float)]():", "func"), + ("class Owner[T: (target, float)]:", "Owner"), + ], +) +@pytest.mark.parametrize("remove", [True, False]) +def test_refuses_changing_bound_or_constraint_binding( + project, declaration, owner, remove +): + attribute = "__constraints__" if "float" in declaration else "__bound__" + expected = "(int, float)" if "float" in declaration else "int" + source = module( + project, + "source", + dedent(f"""\ + original = int + target = original + {declaration} + pass + original = str + print({owner}.__type_params__[0].{attribute} == {expected}) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes( + remove=remove + ) + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.skipif( + sys.version_info < (3, 13), reason="type defaults require Python 3.13+" +) +@pytest.mark.parametrize( + "declaration,owner,initial,expected,rebound", + [ + ("def func[T = target]():", "func", "int", "int", "str"), + ("class Owner[T = target]:", "Owner", "int", "int", "str"), + ("def func[**P = target]():", "func", "[int]", "[int]", "[str]"), + ( + "def func[*Ts = *target]():", + "func", + "tuple[int, float]", + "next(iter(tuple[int, float]))", + "tuple[str, float]", + ), + ], +) +def test_refuses_changing_default_binding( + project, declaration, owner, initial, expected, rebound +): + source = module( + project, + "source", + dedent(f"""\ + original = {initial} + target = original + {declaration} + pass + original = {rebound} + print({owner}.__type_params__[0].__default__ == {expected}) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +def test_refuses_deferring_initializer_side_effect(project): + source = module( + project, + "source", + dedent("""\ + events = [] + def make(): + events.append("called") + return int + target = make() + def func[T: target](): + pass + print(events) + print(func.__type_params__[0].__bound__ is int) + print(events) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "['called']\nTrue\n['called']\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.parametrize("selection", ["definition", "reference"]) +def test_refuses_removing_binding_outside_selected_write_scope(project, selection): + source = module(project, "source", "target = int\n") + annotation = module( + project, + "annotation", + dedent("""\ + import source + class Owner[T: source.target]: + pass + """), + ) + entry = module( + project, + "entry", + dedent("""\ + import source + import annotation + result = source.target + print(result is int, annotation.Owner.__type_params__[0].__bound__ is int) + """), + ) + files = (source, annotation, entry) + before = [resource.read() for resource in files] + output = execute(project, entry) + assert output == "True True\n" + resource = source if selection == "definition" else entry + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline( + project, resource, resource.read().index("target") + 1 + ).get_changes(only_current=selection == "reference", resources=(entry,)) + assert [resource.read() for resource in files] == before + assert execute(project, entry) == output + + +def test_allows_partial_eager_inline_with_type_parameter_binding_retained(project): + source = module( + project, + "source", + dedent("""\ + original = int + target = original + def func[T: target](): + pass + result = target + original = str + print(result is int, func.__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True True\n" + project.do( + create_inline( + project, source, before.index("result = target") + len("result = t") + ).get_changes(only_current=True, remove=False) + ) + assert "target = original" in source.read() + assert "result = original" in source.read() + assert execute(project, source) == output + + +def test_allows_eager_reference_in_generic_function_body(project): + source = module( + project, + "source", + dedent("""\ + target = 42 + def func[T: int](): + return target + print(func(), func.__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "42 True\n" + project.do(create_inline(project, source, before.index("target") + 1).get_changes()) + assert "return 42" in source.read() + assert execute(project, source) == output + + +def test_refuses_removing_captured_local_binding(project): + source = module( + project, + "source", + dedent("""\ + def outer(): + target = int + def func[T: target](): + pass + return func + print(outer().__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +def test_allows_unrelated_shadowed_type_parameter_reference(project): + source = module( + project, + "source", + dedent("""\ + target = 42 + def outer(): + target = int + def func[T: target](): + pass + return func + result = target + print(result, outer().__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "42 True\n" + project.do(create_inline(project, source, before.index("target") + 1).get_changes()) + assert "result = 42" in source.read() + assert execute(project, source) == output + + +@pytest.mark.parametrize( + "definition,owner", + [ + ( + dedent("""\ + def func[T: target](target): + pass + """), + "func", + ), + ( + dedent("""\ + async def func[T: target](target): + pass + """), + "func", + ), + ( + dedent("""\ + def func[T: target](): + target = str + """), + "func", + ), + ( + dedent("""\ + class Owner[T: target]: + target = str + """), + "Owner", + ), + ], +) +def test_refuses_dependency_hidden_by_function_or_class_body( + project, definition, owner +): + source = module( + project, + "source", + dedent("original = int\ntarget = original\n") + definition + dedent(f"""\ + original = str + print({owner}.__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.parametrize( + "definition,owner", + [ + ( + dedent("""\ + def func[T: target](target): + pass + """), + "Outer.func", + ), + ( + dedent("""\ + class Owner[T: target]: + target = str + """), + "Outer.Owner", + ), + ], +) +def test_refuses_removing_enclosing_class_namespace_binding(project, definition, owner): + code = dedent("class Outer:\n target = int\n") + "".join( + " " + line for line in definition.splitlines(True) + ) + code += f"print({owner}.__type_params__[0].__bound__ is int)\n" + source = module(project, "source", code) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.parametrize( + "declaration,expected", + [ + ("def func[target: target]():", "func.__type_params__[0]"), + ("def func[target, T: target]():", "func.__type_params__[0]"), + ], +) +def test_allows_removing_binding_shadowed_by_type_parameter( + project, declaration, expected +): + index = 1 if ", T" in declaration else 0 + source = module( + project, + "source", + dedent(f"""\ + target = int + {declaration} + pass + result = target + print(result is int, func.__type_params__[{index}].__bound__ is {expected}) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True True\n" + project.do( + create_inline( + project, source, before.index("result = target") + len("result = t") + ).get_changes(only_current=True) + ) + assert "result = int" in source.read() + assert execute(project, source) == output + + +@pytest.mark.skipif( + sys.version_info < (3, 13), reason="type defaults require Python 3.13+" +) +@pytest.mark.parametrize( + "definition,owner", + [ + ( + dedent("""\ + def func[T = target](target): + pass + """), + "func", + ), + ( + dedent("""\ + class Owner[T = target]: + target = str + """), + "Owner", + ), + ], +) +def test_refuses_default_dependency_hidden_by_declared_scope( + project, definition, owner +): + source = module( + project, + "source", + dedent("original = int\ntarget = original\n") + definition + dedent(f"""\ + original = str + print({owner}.__type_params__[0].__default__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="nested annotation scopes in classes require Python 3.14+", +) +@pytest.mark.parametrize( + "expression", + [ + "(lambda: target)()", + "[target for unused in [0]][0]", + "next(target for unused in [0])", + "{unused: target for unused in [0]}[0]", + "{target for unused in [0]}.pop()", + ], +) +def test_refuses_inner_expression_resolving_module_namespace(project, expression): + source = module( + project, + "source", + dedent(f"""\ + original = int + target = original + class Outer: + target = bytes + def func[T: {expression}](target): + pass + original = str + print(Outer.func.__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.parametrize( + "expression", + [ + "(lambda target: target)(int)", + "[target for target in [int]][0]", + "next(target for target in [int])", + "{target: target for target in [int]}[int]", + "{target for target in [int]}.pop()", + ], +) +def test_allows_removing_binding_shadowed_by_inner_expression(project, expression): + source = module( + project, + "source", + dedent(f"""\ + target = 42 + def func[T: {expression}](): + pass + result = target + print(result, func.__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "42 True\n" + project.do( + create_inline( + project, source, before.index("result = target") + len("result = t") + ).get_changes(only_current=True) + ) + assert "result = 42" in source.read() + assert execute(project, source) == output + + +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="nested annotation scopes in classes require Python 3.14+", +) +@pytest.mark.parametrize( + "expression", + [ + "(lambda value=target: value)()", + "[value for value in [target]][0]", + ], +) +def test_refuses_inner_expression_outer_class_namespace_dependency(project, expression): + source = module( + project, + "source", + dedent(f"""\ + class Outer: + target = int + def func[T: {expression}](target): + pass + print(Outer.func.__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.parametrize( + "definition,comparison", + [ + ( + dedent("""\ + def outer[target](): + def func[T: target](): + pass + return func.__type_params__[0].__bound__ is outer.__type_params__[0] + """), + "outer()", + ), + ( + dedent("""\ + class Outer[target]: + def func[T: target](): + pass + """), + "Outer.func.__type_params__[0].__bound__ is Outer.__type_params__[0]", + ), + ( + dedent("""\ + class Outer[target]: + target = bytes + def method(self): + def func[T: target](): + pass + return func.__type_params__[0].__bound__ is Outer.__type_params__[0] + """), + "Outer().method()", + ), + ], +) +def test_allows_removing_binding_shadowed_by_enclosing_type_parameter( + project, definition, comparison +): + source = module( + project, + "source", + "target = 42\n" + definition + dedent(f"""\ + result = target + print(result, {comparison}) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "42 True\n" + project.do( + create_inline( + project, source, before.index("result = target") + len("result = t") + ).get_changes(only_current=True) + ) + assert "result = 42" in source.read() + assert execute(project, source) == output + + +def test_refuses_local_binding_inside_enclosing_generic_function(project): + source = module( + project, + "source", + dedent("""\ + def outer[target](): + def inner(): + target = int + def func[T: target](): + pass + return func + return inner() + print(outer().__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target = int") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.parametrize( + "initializer,assignment", + [ + ("Box()", "target.attr"), + ("{}", "target[0]"), + ], +) +def test_refuses_object_dependency_in_comprehension_assignment_target( + project, initializer, assignment +): + source = module( + project, + "source", + dedent(f"""\ + class Box: + pass + target = {initializer} + def func[T: [int for {assignment} in [int]][0]](target): + pass + print(func.__type_params__[0].__bound__ is int) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "True\n" + with pytest.raises(RefactoringError, match="deferred annotation"): + create_inline(project, source, before.index("target") + 1).get_changes() + assert source.read() == before + assert execute(project, source) == output + + +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="nested annotation scopes in classes require Python 3.14+", +) +@pytest.mark.parametrize( + "expression", + [ + "(lambda: target)()", + "[target for unused in [0]][0]", + ], +) +def test_allows_inner_expression_capturing_enclosing_class_type_parameter( + project, expression +): + source = module( + project, + "source", + dedent(f"""\ + target = 42 + class Owner[target]: + target = bytes + def func[T: {expression}](): + pass + result = target + print(result, Owner.func.__type_params__[0].__bound__ is Owner.__type_params__[0]) + """), + ) + before = source.read() + output = execute(project, source) + assert output == "42 True\n" + project.do( + create_inline( + project, source, before.index("result = target") + len("result = t") + ).get_changes(only_current=True) + ) + assert "result = 42" in source.read() + assert execute(project, source) == output diff --git a/ropetest/refactor/inline_unused_initializer_test.py b/ropetest/refactor/inline_unused_initializer_test.py new file mode 100644 index 000000000..ce23dcf2a --- /dev/null +++ b/ropetest/refactor/inline_unused_initializer_test.py @@ -0,0 +1,422 @@ +import os +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base.exceptions import RefactoringError +from rope.base.project import Project +from rope.refactor.inline import create_inline + +NAME_SOURCE = dedent("""\ + def run(): + try: + target = missing + print("body") + except (NameError, TypeError, ValueError): + print("caught") + run() + print("end") +""") + + +def _run(root): + env = os.environ.copy() + env.pop("PYTHONPATH", None) + result = subprocess.run( + [sys.executable, "-B", "-m", "main"], + cwd=root, + env=env, + capture_output=True, + text=True, + timeout=15, + ) + return result.returncode, result.stdout, result.stderr + + +def _exercise(tmp_path, sources, expected, refuse=False, selected=None, **options): + for name, code in sources.items(): + (tmp_path / name).write_text(code, encoding="utf-8") + (tmp_path / "sentinel.txt").write_text("untouched\n", encoding="utf-8") + before = {p.name: p.read_bytes() for p in tmp_path.iterdir() if p.is_file()} + assert _run(tmp_path) == (0, expected, "") + project = Project( + tmp_path, + save_history=False, + save_objectdb=False, + validate_objectdb=False, + automatic_soa=False, + import_dynload_stdmods=False, + ) + try: + resource = project.get_file("subject.py") + code = resource.read() + offset = code.index("target") + 1 + if selected == "last_read": + offset = code.rindex("target") + 1 + if "resources" in options: + options["resources"] = [ + project.get_file(name) for name in options["resources"] + ] + try: + changes = create_inline(project, resource, offset).get_changes(**options) + except RefactoringError: + if not refuse: + raise + assert before == {name: (tmp_path / name).read_bytes() for name in before} + assert _run(tmp_path) == (0, expected, "") + return + assert before == {name: (tmp_path / name).read_bytes() for name in before} + project.do(changes) + after = _run(tmp_path) + changed = { + name: (tmp_path / name).read_text(encoding="utf-8") for name in sources + } + assert after == (0, expected, ""), {"after": after, "sources": changed} + assert not refuse, {"unsafe_removal_accepted": changed, "after": after} + assert (tmp_path / "sentinel.txt").read_text() == "untouched\n" + return changed + finally: + project.close() + + +def _single(tmp_path, code, expected, **options): + return _exercise( + tmp_path, + {"subject.py": code, "main.py": "import subject\n"}, + expected, + **options, + ) + + +def test_unused_unknown_name_removal_is_atomic(tmp_path): + _single(tmp_path, NAME_SOURCE, "caught\nend\n", refuse=True) + + +def test_unused_unknown_name_remove_false_keeps_exception(tmp_path): + changed = _single(tmp_path, NAME_SOURCE, "caught\nend\n", remove=False) + assert changed["subject.py"] == NAME_SOURCE + + +@pytest.mark.parametrize("initializer", ["42", "[1, (-2, {'key': {1, 2}})]"]) +def test_unused_literal_removal_preserves_execution(tmp_path, initializer): + changed = _single( + tmp_path, NAME_SOURCE.replace("missing", initializer), "body\nend\n" + ) + assert "target =" not in changed["subject.py"] + + +@pytest.mark.parametrize("initializer", ["[missing]", "{[]: 42}"]) +def test_unused_container_removal_keeps_lookup_or_hash_failure(tmp_path, initializer): + _single( + tmp_path, + NAME_SOURCE.replace("missing", initializer), + "caught\nend\n", + refuse=True, + ) + + +def test_unused_shadowed_empty_set_call_is_not_a_literal(tmp_path): + code = dedent("""\ + def set(): + print('called') + return [] + target = set() + print('end') + """) + _single(tmp_path, code, "called\nend\n", refuse=True) + + +def test_used_name_initializer_is_still_inlined(tmp_path): + code = dedent("""\ + value = 42 + target = value + print(target) + """) + changed = _single(tmp_path, code, "42\n") + assert "target =" not in changed["subject.py"] + assert "print(value)" in changed["subject.py"] + + +def test_only_current_remove_false_keeps_other_read_and_definition(tmp_path): + code = "target = 42\nprint(target)\nprint(target)\n" + changed = _single( + tmp_path, + code, + "42\n42\n", + selected="last_read", + only_current=True, + remove=False, + ) + assert changed["subject.py"] == "target = 42\nprint(target)\nprint(42)\n" + + +def test_only_current_removal_with_another_read_is_atomic(tmp_path): + code = "target = 42\nprint(target)\nprint(target)\n" + _single( + tmp_path, code, "42\n42\n", selected="last_read", only_current=True, refuse=True + ) + + +def test_only_current_removal_with_one_read_is_still_supported(tmp_path): + changed = _single( + tmp_path, + "target = 42\nprint(target)\n", + "42\n", + selected="last_read", + only_current=True, + ) + assert changed["subject.py"] == "print(42)\n" + + +def _imported_sources(): + return { + "subject.py": "value = 42\ntarget = value\n", + "main.py": "from subject import target\nprint(target)\n", + } + + +def test_actual_importer_read_allows_name_initializer_removal(tmp_path): + changed = _exercise(tmp_path, _imported_sources(), "42\n") + assert "target =" not in changed["subject.py"] + + +def test_resources_excluding_a_read_cannot_remove_definition(tmp_path): + _exercise( + tmp_path, _imported_sources(), "42\n", resources=["subject.py"], refuse=True + ) + + +def test_resources_excluding_definition_leave_initializer_in_place(tmp_path): + sources = _imported_sources() + changed = _exercise(tmp_path, sources, "42\n", resources=["main.py"]) + assert changed["subject.py"] == sources["subject.py"] + + +@pytest.mark.parametrize("annotation", ["missing", "record()"]) +def test_module_annotation_cannot_be_discarded_with_literal_initializer( + tmp_path, annotation +): + code = dedent("""\ + def record(): + print('annotation') + return int + try: + target: ANNOTATION = 42 + print('body') + except NameError: + print('caught') + """).replace("ANNOTATION", annotation) + main = dedent("""\ + import subject + try: + print(subject.__annotations__['target']) + except (NameError, KeyError): + print('annotation caught') + """) + # Read the actual annotation dictionary, including lazy annotations on 3.14. + if annotation == "missing": + expected = ( + "caught\nannotation caught\n" + if sys.version_info < (3, 14) + else "body\nannotation caught\n" + ) + else: + expected = ( + "annotation\nbody\n\n" + if sys.version_info < (3, 14) + else "body\nannotation\n\n" + ) + _exercise(tmp_path, {"subject.py": code, "main.py": main}, expected, refuse=True) + + +def test_function_local_annotation_does_not_block_pure_unused_initializer(tmp_path): + code = NAME_SOURCE.replace("target = missing", "target: missing = 42") + changed = _single(tmp_path, code, "body\nend\n") + assert "target:" not in changed["subject.py"] + + +def test_docs_occurrence_is_not_a_runtime_read(tmp_path): + code = NAME_SOURCE.replace('print("body")', 'print("body") # target') + _single(tmp_path, code, "caught\nend\n", docs=True, refuse=True) + + +@pytest.mark.parametrize("remove", [True, False]) +def test_dormant_load_does_not_erase_initializer_evaluation(tmp_path, remove): + code = dedent("""\ + def run(): + try: + target = missing + if False: + print(target) + print("body") + except (NameError, TypeError, ValueError): + print("caught") + run() + print("end") + """) + _single(tmp_path, code, "caught\nend\n", remove=remove, refuse=remove) + + +def test_used_unknown_name_keeps_exception_before_intervening_print(tmp_path): + code = NAME_SOURCE.replace('print("body")', 'print("body")\n print(target)') + _single(tmp_path, code, "caught\nend\n", refuse=True) + + +def test_used_module_annotation_keeps_observed_effect(tmp_path): + code = dedent("""\ + import sys + import typing + def record(): + print('annotation') + return int + target: record() = 42 + print(target) + typing.get_type_hints(sys.modules[__name__]) + print('end') + """) + expected = ( + "annotation\n42\nend\n" + if sys.version_info < (3, 14) + else "42\nannotation\nend\n" + ) + _single(tmp_path, code, expected, refuse=True) + + +def test_used_name_bound_in_outer_module_is_still_inlined(tmp_path): + code = dedent("""\ + value = 42 + def run(): + target = value + print(target) + run() + """) + changed = _single(tmp_path, code, "42\n") + assert "target =" not in changed["subject.py"] + assert "print(value)" in changed["subject.py"] + + +def test_unused_bound_name_can_be_removed_without_discarding_an_effect(tmp_path): + code = "value = 42\ntarget = value\nprint('end')\n" + changed = _single(tmp_path, code, "end\n") + assert changed["subject.py"] == "value = 42\nprint('end')\n" + + +def test_deleted_outer_name_cannot_be_discarded_with_unused_initializer(tmp_path): + code = dedent("""\ + value = 42 + def run(): + try: + target = value + print('body') + except NameError: + print('caught') + del value + run() + print('end') + """) + _single(tmp_path, code, "caught\nend\n", refuse=True) + + +def test_unrelated_local_deletion_does_not_block_outer_name_initializer(tmp_path): + code = dedent("""\ + value = 42 + def unrelated(): + value = 0 + del value + def run(): + target = value + print(target) + unrelated() + run() + """) + changed = _single(tmp_path, code, "42\n") + assert "target =" not in changed["subject.py"] + assert "print(value)" in changed["subject.py"] + + +def test_builtin_name_alias_is_still_inlined(tmp_path): + code = "target = len\nprint(target([1, 2]))\n" + changed = _single(tmp_path, code, "2\n") + assert changed["subject.py"] == "print(len([1, 2]))\n" + assert changed["main.py"] == "import subject\n" + + +def test_function_parameter_alias_is_still_inlined(tmp_path): + code = dedent("""\ + def run(value): + target = value + print(target) + run(42) + """) + changed = _single(tmp_path, code, "42\n") + assert changed["subject.py"] == dedent("""\ + def run(value): + print(value) + run(42) + """) + assert changed["main.py"] == "import subject\n" + + +def test_same_module_function_alias_is_still_inlined(tmp_path): + code = dedent("""\ + def value(): + return 42 + target = value + print(target()) + """) + changed = _single(tmp_path, code, "42\n") + assert changed["subject.py"] == dedent("""\ + def value(): + return 42 + print(value()) + """) + assert changed["main.py"] == "import subject\n" + + +def test_imported_module_alias_is_still_inlined(tmp_path): + code = "import math as value\ntarget = value\nprint(target.sqrt(4))\n" + changed = _single(tmp_path, code, "2.0\n") + assert changed["subject.py"] == "import math as value\nprint(value.sqrt(4))\n" + assert changed["main.py"] == "import subject\n" + + +def test_imported_value_alias_is_still_inlined(tmp_path): + code = "from math import pi as value\ntarget = value\nprint(target)\n" + changed = _single(tmp_path, code, "3.141592653589793\n") + assert changed["subject.py"] == "from math import pi as value\nprint(value)\n" + assert changed["main.py"] == "import subject\n" + + +def test_outer_name_remains_bound_across_unrelated_literal_and_pass(tmp_path): + code = dedent("""\ + value = 42 + unrelated = 0 + pass + def run(): + target = value + print(target) + run() + """) + changed = _single(tmp_path, code, "42\n") + assert changed["subject.py"] == dedent("""\ + value = 42 + unrelated = 0 + pass + def run(): + print(value) + run() + """) + assert changed["main.py"] == "import subject\n" + + +def test_class_name_alias_removal_is_conservatively_atomic(tmp_path): + code = dedent("""\ + class Box: + value = 42 + target = value + print(target) + print(Box.value) + """) + _single(tmp_path, code, "42\n42\n", refuse=True) diff --git a/ropetest/refactor/inlinetest.py b/ropetest/refactor/inlinetest.py index 8ded0143c..63dc12e47 100644 --- a/ropetest/refactor/inlinetest.py +++ b/ropetest/refactor/inlinetest.py @@ -37,11 +37,132 @@ def test_simple_case(self): refactored = self._inline(code, code.index("a_var") + 1) self.assertEqual("another_var = 10\n", refactored) + def test_inline_assignment_before_another_statement(self): + code = dedent("""\ + value = "text"; other = 10 + print(other, value) + """) + refactored = self._inline(code, code.index("value") + 1) + self.assertEqual('other = 10\nprint(other, "text")\n', refactored) + + def test_inline_first_target_of_chained_assignment(self): + code = dedent("""\ + value = other = 10 + print(other, value) + """) + refactored = self._inline(code, code.index("value") + 1) + self.assertEqual("other = 10\nprint(other, 10)\n", refactored) + + def test_inline_parenthesized_first_target_of_chained_assignment(self): + code = dedent("""\ + (value) = other = 10 + print(other, value) + """) + refactored = self._inline(code, code.index("value") + 1) + self.assertEqual("other = 10\nprint(other, 10)\n", refactored) + + def test_inline_parenthesized_last_target_of_chained_assignment(self): + code = dedent("""\ + value = (other) = 10 + print(value, other) + """) + refactored = self._inline(code, code.index("other") + 1) + self.assertEqual("value = 10\nprint(value, 10)\n", refactored) + + def test_inline_annotated_assignment(self): + # A module-level ``AnnAssign`` publishes its annotation in the + # module ``__annotations__`` mapping, so discarding the statement + # while inlining would change runtime-observable state (#903). + code = dedent("""\ + value: int = 10 + print(value) + """) + with self.assertRaises(rope.base.exceptions.RefactoringError): + self._inline(code, code.index("value") + 1) + + def test_inline_annotated_assignment_without_removal_keeps_statement(self): + code = dedent("""\ + value: int = 10 + print(value) + """) + refactored = self._inline( + code, code.rindex("value") + 1, remove=False, only_current=True + ) + self.assertEqual("value: int = 10\nprint(10)\n", refactored) + + def test_inline_attribute_before_same_named_local_assignment(self): + code = dedent("""\ + class Example: + def method(self): + self.value = 10; value = 20 + print(self.value, value) + """) + refactored = self._inline(code, code.index("value") + 1) + self.assertEqual( + dedent("""\ + class Example: + def method(self): + value = 20 + print(10, value) + """), + refactored, + ) + + def test_inline_local_after_same_named_attribute_in_chained_assignment(self): + code = dedent("""\ + class Example: + def method(self): + self.value = value = 10 + return self.value, value + """) + local_value = code.index("value", code.index("=")) + refactored = self._inline(code, local_value + 1) + self.assertEqual( + dedent("""\ + class Example: + def method(self): + self.value = 10 + return self.value, 10 + """), + refactored, + ) + def test_empty_case(self): code = "a_var = 10\n" refactored = self._inline(code, code.index("a_var") + 1) self.assertEqual("", refactored) + @testutils.only_for_versions_higher("3.12") + def test_inlining_type_alias_is_refused(self): + code = dedent("""\ + type an_alias = int + value = an_alias.__value__ + """) + with self.assertRaises(rope.base.exceptions.RefactoringError): + self._inline(code, code.index("an_alias") + 1) + + @testutils.only_for_versions_higher("3.12") + def test_inlining_lazily_evaluated_type_alias_is_refused(self): + code = dedent("""\ + type an_alias = Later + def a_func(param: an_alias): + pass + class Later: + pass + """) + with self.assertRaises(rope.base.exceptions.RefactoringError): + self._inline(code, code.index("an_alias") + 1) + + @testutils.only_for_versions_higher("3.12") + def test_inlining_generic_type_alias(self): + code = dedent("""\ + type an_alias[T] = list[T] + def a_func(param: an_alias[int]): + pass + """) + with self.assertRaises(rope.base.exceptions.RefactoringError): + self._inline(code, code.index("an_alias") + 1) + def test_long_definition(self): code = dedent("""\ a_var = 10 + (10 + 10) @@ -113,6 +234,22 @@ def test_on_unknown_vars(self): with self.assertRaises(rope.base.exceptions.RefactoringError): self._inline(code, code.index("another_var") + 1) + @testutils.only_for_versions_higher("3.10") + def test_pattern_capture_cannot_be_inlined(self): + code = dedent("""\ + old_name = 0 + match value: + case old_name: + print(old_name) + print(old_name) + """) + capture_offset = code.index("old_name", code.index("case")) + 1 + with self.assertRaisesRegex( + rope.base.exceptions.RefactoringError, + "Pattern matching captures cannot be inlined", + ): + self._inline(code, capture_offset) + def test_attribute_inlining(self): code = dedent("""\ class A(object): @@ -354,6 +491,26 @@ def a_func(param): self._inline2(self.mod, self.mod.read().index("a_func") + 1) self.assertEqual("print(1)\n", self.mod.read()) + def test_parameters_used_as_subscript_assignment_indices(self): + code = dedent("""\ + registry = {} + def get_registry(): + return registry + def register(name, value): + get_registry()[name] = value + register("cache", 42) + """) + refactored = self._inline(code, code.index("register") + 1) + self.assertEqual( + dedent("""\ + registry = {} + def get_registry(): + return registry + get_registry()["cache"] = 42 + """), + refactored, + ) + def test_parameters_renaming_for_passed_statements(self): self.mod.write(dedent("""\ def a_func(param): diff --git a/ropetest/refactor/instance_method_transfertest.py b/ropetest/refactor/instance_method_transfertest.py new file mode 100644 index 000000000..1347b8f7b --- /dev/null +++ b/ropetest/refactor/instance_method_transfertest.py @@ -0,0 +1,852 @@ +from __future__ import annotations + +import importlib +import sys +import unittest +from textwrap import dedent + +from rope.base import exceptions +from rope.refactor import move +from ropetest import testutils + + +class InstanceMethodTransferTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.project = testutils.sample_project() + self.handlers = testutils.create_module(self.project, "handlers") + self.source = testutils.create_module(self.project, "source") + self.destination = testutils.create_module(self.project, "destination") + self.handlers.write( + dedent("""\ + class Handlers: + @staticmethod + def handler(command): + def decorate(func): + if hasattr(func, "_handler_config"): + raise ValueError("duplicate") + setattr(func, "_handler_config", command) + return func + return decorate + + h = Handlers + """) + ) + self.destination.write( + dedent("""\ + class Target: + queue_service: object + """) + ) + + def tearDown(self) -> None: + testutils.remove_project(self.project) + super().tearDown() + + def _move(self): + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return self.queue.execute(request) + """) + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + self.assertIsInstance(operation, move.MoveMethod) + changes = operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[self.handlers, self.source, self.destination], + ) + self.project.do(changes) + + def _load_runtime_module(self, name: str): + module_names = ("handlers", name) + prior = {module: sys.modules.pop(module, None) for module in module_names} + sys.path.insert(0, self.project.address) + try: + return importlib.import_module(name) + finally: + sys.path.remove(self.project.address) + for module, previous in prior.items(): + sys.modules.pop(module, None) + if previous is not None: + sys.modules[module] = previous + + def test_transfers_handler_with_port_rewrite_without_forwarding(self) -> None: + self._move() + self.assertNotIn("def execute", self.source.read()) + self.assertIn("@h.handler(command=int)", self.destination.read()) + self.assertIn( + "self.queue_service.queue.execute(request)", self.destination.read() + ) + self.assertNotIn("return self.", self.source.read()) + destination = self._load_runtime_module("destination") + self.assertIs(destination.Target.execute._handler_config, int) + + class Queue: + @staticmethod + def execute(request): + return request + 1 + + class QueuePort: + queue = Queue() + + target = destination.Target() + target.queue_service = QueuePort() + self.assertEqual(target.execute(4), 5) + + def test_same_module_transfer_keeps_local_dependencies_without_self_import( + self, + ) -> None: + source = dedent("""\ + def handler(command): + def decorate(func): + setattr(func, "_handler_config", command) + return func + return decorate + + class Target: + queue_service: object + + class Legacy: + @handler(command=int) + def execute(self, request): + return self.queue.execute(request) + """) + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + changes = operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.source, + source.index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[self.source], + ) + self.project.do(changes) + self.assertNotIn("import", self.source.read()) + module = self._load_runtime_module("source") + self.assertIs(module.Target.execute._handler_config, int) + + def test_same_module_transfer_from_function_local_class(self) -> None: + source = dedent("""\ + from handlers import h + + class Target: + queue_service: object + + def build(): + class Legacy: + @h.handler(command=int) + def execute(self, request): + return self.queue.execute(request) + return Legacy + """) + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + changes = operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.source, + source.index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[self.handlers, self.source], + ) + self.project.do(changes) + module = self._load_runtime_module("source") + self.assertIs(module.Target.execute._handler_config, int) + self.assertFalse(hasattr(module.build(), "execute")) + + def test_transfer_applies_the_requested_method_name(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return self.queue.execute(request) + """) + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + changes = operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "run")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[self.handlers, self.source, self.destination], + ) + self.project.do(changes) + destination = self._load_runtime_module("destination") + self.assertIs(destination.Target.run._handler_config, int) + self.assertFalse(hasattr(destination.Target, "execute")) + + def test_refuses_method_private_name_mangling_after_runtime_proof(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + def __init__(self): + self.__secret = "preserved" + + @h.handler(command=int) + def execute(self, request): + return self.__secret + """) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + self.assertEqual(legacy().execute(1), "preserved") + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "Private identifier changes" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_decorator_private_binding_after_runtime_proof(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + __Command = int + + @h.handler(command=__Command) + def execute(self, request): + return request + """) + self.destination.write( + self.destination.read().replace( + "class Target:\n queue_service: object", + "class Target:\n __Command = str\n queue_service: object", + ) + ) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + self.assertIs(legacy.execute._handler_config, int) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "Private identifier changes" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_annotation_private_binding_after_runtime_proof(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + __Request = int + + @h.handler(command=int) + def execute(self, request: __Request): + return request + """) + self.destination.write( + self.destination.read().replace( + "class Target:\n queue_service: object", + "class Target:\n __Request = str\n queue_service: object", + ) + ) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + self.assertIs(legacy.execute.__annotations__["request"], int) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "Private identifier changes" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_class_scope_default_and_annotation_binding_changes(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + TYPE = int + DEFAULT = 7 + + @h.handler(command=int) + def execute(self, request: TYPE = DEFAULT): + return request + """) + self.destination.write( + dedent("""\ + class Target: + TYPE = str + DEFAULT = "destination" + queue_service: object + """) + ) + naive = testutils.create_module(self.project, "naive") + naive.write( + dedent("""\ + from handlers import h + + class Target: + TYPE = str + DEFAULT = "destination" + + @h.handler(command=int) + def execute(self, request: TYPE = DEFAULT): + return request + """) + ) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + naive_target = self._load_runtime_module("naive").Target + self.assertEqual(legacy.execute.__defaults__, (7,)) + self.assertIs(legacy.execute.__annotations__["request"], int) + self.assertEqual(naive_target.execute.__defaults__, ("destination",)) + self.assertIs(naive_target.execute.__annotations__["request"], str) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "signature binding changed" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_custom_descriptor_default_after_runtime_proof(self) -> None: + self.handlers.write( + self.handlers.read() + + dedent("""\ + class Descriptor: + def __init__(self): + self.reads = 0 + def __get__(self, instance, owner): + self.reads += 1 + return self.reads + class Settings: + value = Descriptor() + """) + ) + source = dedent("""\ + from handlers import h, Settings + class Legacy: + @h.handler(command=int) + def execute(self, request=Settings.value): + return request + """) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + self.assertEqual(legacy.execute.__defaults__, (1,)) + original_destination = self.destination.read() + with self.assertRaisesRegex( + exceptions.RefactoringError, "attribute may invoke descriptor behavior" + ): + move.MoveMethod.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + self.assertEqual(self.source.read(), source) + self.assertEqual(self.destination.read(), original_destination) + + def test_refuses_decorator_that_wraps_the_method(self) -> None: + self.handlers.write( + dedent("""\ + class Handlers: + @staticmethod + def handler(command): + def decorate(func): + def wrapped(self, request): + return func(self, request) + return wrapped + return decorate + + h = Handlers + """) + ) + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return self.queue.execute(request) + """) + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "original method" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_identity_decorator_with_conditional_return(self) -> None: + self.handlers.write( + dedent("""\ + class Handlers: + @staticmethod + def handler(command): + def decorate(func): + if hasattr(func, "_handler_config"): + raise ValueError("duplicate") + setattr(func, "_handler_config", command) + if command is int: + return func + return decorate + + h = Handlers + """) + ) + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return request + """) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + self.assertEqual(legacy().execute(4), 4) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "explicit return" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_identity_decorator_with_unproven_helper(self) -> None: + self.handlers.write( + dedent("""\ + def add_metadata(func, command): + setattr(func, "_handler_config", command) + + class Handlers: + @staticmethod + def handler(command): + def decorate(func): + add_metadata(func, command) + return func + return decorate + + h = Handlers + """) + ) + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return request + """) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + self.assertEqual(legacy().execute(4), 4) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "Identity decorator" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_same_named_local_metadata_constructor_with_runtime_effect(self) -> None: + self.handlers.write( + dedent("""\ + class DecoratorConfig: + def __init__(self, func, command): + if func.__qualname__.startswith("Target."): + func.__code__ = ( + lambda self, request: "changed" + ).__code__ + self.command = command + + class Handlers: + @staticmethod + def handler(command): + def decorate(func): + settings = DecoratorConfig(func, command) + setattr(func, "_handler_config", settings.command) + return func + return decorate + + h = Handlers + """) + ) + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return request + """) + naive = testutils.create_module(self.project, "naive") + naive.write( + dedent("""\ + from handlers import h + + class Target: + @h.handler(command=int) + def execute(self, request): + return request + """) + ) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + target = self._load_runtime_module("naive").Target + self.assertEqual(legacy().execute(4), 4) + self.assertEqual(target().execute(4), "changed") + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "canonical source identity" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_decorator_argument_with_caller_dependent_effect(self) -> None: + self.handlers.write( + dedent("""\ + import inspect + + def choose(): + caller = inspect.currentframe().f_back.f_globals["__name__"] + return int if caller == "source" else str + + class Handlers: + @staticmethod + def handler(command): + def decorate(func): + setattr(func, "_handler_config", command) + return func + return decorate + + h = Handlers + """) + ) + source = dedent("""\ + from handlers import choose, h + + class Legacy: + @h.handler(command=choose()) + def execute(self, request): + return request + """) + naive = testutils.create_module(self.project, "naive") + naive.write( + dedent("""\ + from handlers import choose, h + + class Target: + @h.handler(command=choose()) + def execute(self, request): + return request + """) + ) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + target = self._load_runtime_module("naive").Target + self.assertIs(legacy.execute._handler_config, int) + self.assertIs(target.execute._handler_config, str) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "handler decorator.*effects" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_decorator_attribute_with_custom_getattribute_effect(self) -> None: + self.handlers.write( + dedent("""\ + import inspect + + class DynamicMeta(type): + def __getattribute__(cls, name): + if name == "command": + caller = inspect.currentframe().f_back.f_globals["__name__"] + return int if caller == "source" else str + return super().__getattribute__(name) + + class Selector(metaclass=DynamicMeta): + command = int + + class Handlers: + @staticmethod + def handler(command): + def decorate(func): + setattr(func, "_handler_config", command) + return func + return decorate + + h = Handlers + """) + ) + source = dedent("""\ + from handlers import Selector, h + + class Legacy: + @h.handler(command=Selector.command) + def execute(self, request): + return request + """) + naive = testutils.create_module(self.project, "naive") + naive.write( + dedent("""\ + from handlers import Selector, h + + class Target: + @h.handler(command=Selector.command) + def execute(self, request): + return request + """) + ) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + target = self._load_runtime_module("naive").Target + self.assertIs(legacy.execute._handler_config, int) + self.assertIs(target.execute._handler_config, str) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "native __getattribute__" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_future_annotations_policy_change_after_runtime_proof(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + TYPE = int + + @h.handler(command=int) + def execute(self, request: TYPE): + return request + """) + self.destination.write( + dedent("""\ + from __future__ import annotations + + class Target: + TYPE = str + queue_service: object + """) + ) + naive = testutils.create_module(self.project, "naive") + naive.write( + dedent("""\ + from __future__ import annotations + from handlers import h + + class Target: + TYPE = str + + @h.handler(command=int) + def execute(self, request: TYPE): + return request + """) + ) + self.source.write(source) + legacy = self._load_runtime_module("source").Legacy + naive_target = self._load_runtime_module("naive").Target + self.assertIs(legacy.execute.__annotations__["request"], int) + self.assertEqual(naive_target.execute.__annotations__["request"], "TYPE") + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "future policy changes" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_unmapped_receiver_before_planning_changes(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return self.queue.execute(self.other, request) + """) + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "no declared port mapping" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[self.handlers, self.source, self.destination], + ) + + def test_refuses_external_method_references(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + @h.handler(command=int) + def execute(self, request): + return self.queue.execute(request) + + operation = Legacy().execute(1) + """) + caller = testutils.create_module(self.project, "caller") + caller.write("from source import Legacy\nvalue = Legacy().execute(1)\n") + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "external method reference" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[ + self.handlers, + self.source, + self.destination, + caller, + ], + ) + + def test_refuses_decorator_binding_changed_by_destination_class(self) -> None: + source = dedent("""\ + from handlers import h + + class Legacy: + COMMAND = int + + @h.handler(command=COMMAND) + def execute(self, request): + return self.queue.execute(request) + """) + self.destination.write( + self.destination.read().replace( + "class Target:\n queue_service: object", + "class Target:\n COMMAND = str\n queue_service: object", + ) + ) + self.source.write(source) + operation = move.create_move( + self.project, self.source, source.index("execute(self") + 1 + ) + with self.assertRaisesRegex( + exceptions.RefactoringError, "decorator binding changed" + ): + operation.get_instance_class_changes( + self.project, + [(self.source, source.index("execute(self") + 1, "execute")], + self.destination, + self.destination.read().index("Target") + 1, + receiver_rewrites={"queue": "queue_service.queue"}, + resources=[self.handlers, self.source, self.destination], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ropetest/refactor/move_public_access_test.py b/ropetest/refactor/move_public_access_test.py new file mode 100644 index 000000000..5b8e03c5f --- /dev/null +++ b/ropetest/refactor/move_public_access_test.py @@ -0,0 +1,230 @@ +import os +import subprocess +import sys +from textwrap import dedent, indent + +import pytest + +from rope.base import evaluate, exceptions +from rope.base.project import Project +from rope.refactor import move + + +@pytest.fixture +def native_project(tmp_path): + project = Project(tmp_path, ropefolder=None, source_folders=[""]) + yield project + project.close() + + +def _write(project, path, source): + target = project.root.real_path + "/" + path + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w", encoding="utf-8") as output: + output.write(source) + return project.get_file(path) + + +def _run(project, code): + result = subprocess.run( + [sys.executable, "-B", "-c", code], + cwd=project.root.real_path, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +def test_move_to_same_holder_normalizes_self_import_and_evaluated_base(native_project): + project = native_project + project.prefs.imports.preferred_import_style = "from-global" + target = _write(project, "target.py", dedent("""\ + class Destination: + class First: + def kind(self): + return "first" + """)) + source = _write(project, "source.py", dedent("""\ + from __future__ import annotations + from target import Destination as Existing + class Owner: + class Second(Existing.First): + def kind(self): + return "second:" + super().kind() + def first(self) -> 'Existing.First': + return Existing.First() + """)) + mover = move.create_move(project, source, source.read().index("Second")) + project.do(mover.get_changes(target, destination_offset=target.read().index("Destination"))) + assert "from target import" not in target.read() + assert _run(project, dedent("""\ + from typing import get_type_hints + from target import Destination + assert Destination.Second.__mro__[1] is Destination.First + assert get_type_hints(Destination.Second.first)["return"] is Destination.First + assert isinstance(Destination.Second().first(), Destination.First) + print(Destination.Second().kind()) + """)) == "second:first\n" + + +def test_mutual_nested_contracts_and_sibling_inheritance_move_as_a_cohort(native_project): + project = native_project + project.prefs.imports.preferred_import_style = "from-global" + target = _write(project, "target.py", dedent("""\ + class Destination: + pass + """)) + source = _write(project, "source.py", dedent("""\ + from __future__ import annotations + class Owner: + class First: + def other(self) -> 'Owner.Second': + return Owner.Second() + class Second(First): + def first(self) -> 'Owner.First': + return Owner.First() + """)) + for name in ("First", "Second"): + mover = move.create_move(project, source, source.read().index("class " + name) + len("class ")) + project.do(mover.get_changes(target, destination_offset=target.read().index("Destination"))) + assert "from target import" not in target.read() + assert "from source import" not in target.read() + assert _run(project, dedent("""\ + from typing import get_type_hints + from target import Destination + assert Destination.Second.__mro__[1] is Destination.First + assert get_type_hints(Destination.First.other)["return"] is Destination.Second + assert get_type_hints(Destination.Second.first)["return"] is Destination.First + assert isinstance(Destination.First().other(), Destination.Second) + assert isinstance(Destination.Second().first(), Destination.First) + print("cohort preserved") + """)) == "cohort preserved\n" + + +def _contract(owner, bases, nesting, kind, *, target=False): + source = "class " + owner + ("(" + ", ".join(bases) + ")" if bases else "") + ":\n" + level = 1 + for number in range(nesting): + source += " " * level + "class Level" + str(number) + source += "(Target):\n" if target and number == nesting - 1 else ":\n" + level += 1 + source += indent(dedent('''\ + class Contract: + def kind(self): + return {!r} + ''').format(kind), " " * level) + return source + + +@pytest.mark.parametrize("namespace_depth", [0, 1, 4]) +@pytest.mark.parametrize("nesting", [0, 1, 3]) +def test_barrels_aliases_inherited_contracts_and_c3_override(native_project, namespace_depth, nesting): + project = native_project + parts = ["ns" + str(number) for number in range(namespace_depth)] + ["contracts"] + directory, package = "/".join(parts), ".".join(parts) + target = _write(project, directory + "/target.py", dedent("""\ + class Target: + pass + """)) + owner = _write(project, directory + "/owner.py", + "from .target import Target\n" + + _contract("Owner", ("Target",) if not nesting else (), nesting, "base", target=True) + + dedent("""\ + class Left(Owner): + pass + class InheritedRight(Owner): + pass + """) + + _contract("Right", ("Owner",), nesting, "override") + + dedent("""\ + class Public(Left, InheritedRight): + pass + class Override(Left, Right): + pass + """) + + _contract("Stranger", (), nesting, "stranger")) + _write(project, directory + "/exports.py", dedent("""\ + from .owner import Public as PublicFacade + Mid = PublicFacade + __all__ = ["PublicFacade", "Mid"] + """)) + _write(project, directory + "/__init__.py", dedent("""\ + from .exports import PublicFacade as p, Mid as Alias + from .owner import Override, Stranger + __all__ = ["p", "Alias", "Override", "Stranger"] + """)) + path = ".".join(["Level" + str(number) for number in range(nesting)] + ["Contract"]) + consumer = _write(project, directory + "/consumer.py", dedent("""\ + from . import * + from .owner import Owner as Direct + from . import exports as namespace + direct = Direct.{path}() + public = p.{path}() + chain = namespace.Mid.{path}() + override = Override.{path}() + stranger = Stranger.{path}() + def annotated(value: 'Alias.{path}') -> 'Alias.{path}': + return value + """).format(path=path)) + script = dedent("""\ + from {package}.consumer import direct, public, chain, override, stranger, annotated + from {package}.owner import Public, Override + assert [item.__name__ for item in Public.__mro__[:4]] == ["Public", "Left", "InheritedRight", "Owner"] + assert [item.__name__ for item in Override.__mro__[:4]] == ["Override", "Left", "Right", "Owner"] + assert type(direct) is type(public) is type(chain) + assert annotated(public) is public + print(direct.kind(), public.kind(), chain.kind(), override.kind(), stranger.kind()) + """).format(package=package) + assert _run(project, script) == "base base base override stranger\n" + public = project.get_module(package + ".owner")["Public"].get_object() + + def selected(site): + environment = site.tools.pymodule.get_source_binding_index().context_at(site.offset).environment + names = site.get_primary_text().split(".") + for count in range(1, len(names)): + binding = evaluate.eval_str(environment, ".".join(names[:count])) + if binding is not None and binding.get_object() is public: + return False + return True + + mover = move.create_move(project, owner, owner.read().index("Contract")) + project.do(mover.get_changes(target, destination_offset=target.read().index("Target"), match_filter=selected)) + assert "public = p." + path + "()" in consumer.read() + assert "chain = namespace.Mid." + path + "()" in consumer.read() + assert "'Alias." + path + "'" in consumer.read() + assert _run(project, script) == "base base base override stranger\n" + + +def test_unresolved_alias_cycle_rejects_before_any_effect(native_project): + project = native_project + source = _write(project, "source.py", dedent("""\ + class Owner: + class Contract: + pass + """)) + target = _write(project, "target.py", dedent("""\ + class Target: + pass + """)) + _write(project, "first.py", "from second import Alias\n") + _write(project, "second.py", "from first import Alias\n") + _write(project, "consumer.py", dedent("""\ + from first import Alias + value: 'Alias.Contract' + """)) + before = source.read(), target.read() + mover = move.create_move(project, source, source.read().index("Contract")) + with pytest.raises(exceptions.RefactoringError, match="Unproven"): + mover.get_changes(target, destination_offset=target.read().index("Target")) + assert (source.read(), target.read()) == before + + +def test_move_global_class_empties_the_old_module(native_project): + source = _write(native_project, "old.py", "class Contract: pass\n") + target = _write(native_project, "new.py", "") + mover = move.MoveClass(native_project, source, source.read().index("Contract")) + native_project.do(mover.get_changes(target)) + assert not source.read() + assert target.read().strip() == "class Contract: pass" diff --git a/ropetest/refactor/movetest.py b/ropetest/refactor/movetest.py index 2e467b82b..ade49a7ca 100644 --- a/ropetest/refactor/movetest.py +++ b/ropetest/refactor/movetest.py @@ -1,11 +1,13 @@ from __future__ import annotations +import runpy +import sys import unittest from textwrap import dedent from typing import TYPE_CHECKING, Union -from rope.base import exceptions -from rope.refactor import move +from rope.base import exceptions, libutils +from rope.refactor import move, occurrences from ropetest import testutils if TYPE_CHECKING: @@ -67,12 +69,666 @@ def _move_to_attr( changes = mover.get_changes(dest_attr, new_name=new_name) self.project.do(changes) + def test_move_nested_class_to_existing_class(self): + self.mod1.write(dedent("""\ + from collections.abc import Mapping + + class Owner: + class Inner: + def size(self, value: Mapping[str, int]) -> int: + return len(value) + """)) + self.mod2.write(dedent("""\ + class Target: + pass + """)) + self.mod3.write(dedent("""\ + from mod1 import Owner + value = Owner.Inner() + """)) + mover = move.create_move( + self.project, self.mod1, self.mod1.read().index("Inner") + ) + changes = mover.get_changes( + self.mod2, destination_offset=self.mod2.read().index("Target") + ) + self.project.do(changes) + self.assertNotIn("class Inner", self.mod1.read()) + self.assertIn("class Inner", self.mod2.read()) + self.assertIn("Mapping", self.mod2.read()) + self.assertIn("mod2.Target.Inner()", self.mod3.read()) + namespace = runpy.run_path(self.mod2.real_path) + self.assertEqual(1, namespace["Target"].Inner().size({"a": 1})) + + def test_move_nested_class_preserves_explicit_namespace_import_root(self): + project = testutils.sample_project(source_folders=[""]) + try: + namespace = project.root.create_folder("infra") + package = testutils.create_package(project, "contracts", namespace) + origin = testutils.create_module(project, "origin", package) + target = testutils.create_module(project, "current", package) + consumer = testutils.create_module(project, "consumer") + origin.write(dedent("""\ + class Owner: + class Inner: + pass + """)) + target.write(dedent("""\ + class Target: + pass + """)) + consumer.write(dedent("""\ + from infra.contracts.origin import Owner + value = Owner.Inner() + """)) + self.assertIs(project.get_module("infra.contracts.current"), project.get_pymodule(target)) + self.assertEqual("infra.contracts.current", libutils.modname(target)) + mover = move.create_move(project, origin, origin.read().index("Inner")) + project.do(mover.get_changes(target, destination_offset=target.read().index("Target"))) + self.assertIn("infra.contracts.current", consumer.read()) + namespace = runpy.run_path(target.real_path) + self.assertIsInstance(namespace["Target"].Inner(), namespace["Target"].Inner) + finally: + testutils.remove_project(project) + + def test_move_nested_class_with_recursive_nested_references(self): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + class Child: + pass + def clone(self): + return Owner.Inner() + def child(self): + return Owner.Inner.Child() + """)) + self.mod2.write(dedent("""\ + class Target: + pass + """)) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + self.project.do(mover.get_changes(self.mod2, destination_offset=self.mod2.read().index("Target"))) + namespace = runpy.run_path(self.mod2.real_path) + inner = namespace["Target"].Inner + self.assertIsInstance(inner().clone(), inner) + self.assertIsInstance(inner().child(), inner.Child) + + def test_move_nested_class_to_module(self): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + pass + """)) + self.mod3.write(dedent("""\ + from mod1 import Owner + value = Owner.Inner() + """)) + self._move(self.mod1, self.mod1.read().index("Inner"), self.mod2) + self.assertIn("mod2.Inner()", self.mod3.read()) + self.assertIn("class Inner:", self.mod2.read()) + compile(self.mod1.read(), self.mod1.path, "exec") + + def test_move_nested_class_in_same_module(self): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + pass + class Target: + pass + value = Owner.Inner() + """)) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + self.project.do(mover.get_changes(self.mod1, destination_offset=self.mod1.read().index("Target"))) + namespace = runpy.run_path(self.mod1.real_path) + self.assertIsInstance(namespace["value"], namespace["Target"].Inner) + + def test_move_nested_class_rejects_enclosing_class_dependency(self): + source = dedent("""\ + class Owner: + class Base: + pass + class Inner(Base): + pass + """) + self.mod1.write(source) + mover = move.create_move(self.project, self.mod1, source.index("Inner")) + with self.assertRaisesRegex(exceptions.RefactoringError, "enclosing"): + mover.get_changes(self.mod2) + self.assertEqual(source, self.mod1.read()) + + def test_move_nested_class_rejects_shadowed_enclosing_dependency(self): + source = dedent("""\ + Base = object + class Owner: + class Base: + pass + class Inner(Base): + pass + """) + self.mod1.write(source) + mover = move.create_move(self.project, self.mod1, source.index("Inner")) + with self.assertRaisesRegex(exceptions.RefactoringError, "enclosing"): + mover.get_changes(self.mod2) + self.assertEqual(source, self.mod1.read()) + + def test_move_decorated_nested_class_copies_imports(self): + self.mod1.write(dedent("""\ + from dataclasses import dataclass + class Owner: + @dataclass + class Inner: + value: int + """)) + self.mod2.write(dedent("""\ + class Target: + pass + """)) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + self.project.do(mover.get_changes(self.mod2, destination_offset=self.mod2.read().index("Target"))) + namespace = runpy.run_path(self.mod2.real_path) + self.assertEqual(3, namespace["Target"].Inner(3).value) + + def test_move_nested_class_rejects_destination_collision(self): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + pass + """)) + destination = dedent("""\ + class Target: + class Inner: + pass + """) + self.mod2.write(destination) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + with self.assertRaisesRegex(exceptions.RefactoringError, "Occupied"): + mover.get_changes(self.mod2, destination_offset=destination.index("Target")) + self.assertEqual(destination, self.mod2.read()) + + def test_move_nested_class_reference_filter_keeps_composed_access(self): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + pass + class Public(Owner): + pass + """)) + self.mod2.write(dedent("""\ + class Target: + pass + """)) + self.mod3.write(dedent("""\ + from mod1 import Owner, Public + direct = Owner.Inner + composed = Public.Inner + direct_annotation: 'Owner.Inner' + composed_annotation: 'Public.Inner' + """)) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + public = self.project.get_pymodule(self.mod1)["Public"].get_object() + changes = mover.get_changes( + self.mod2, + destination_offset=self.mod2.read().index("Target"), + match_filter=lambda occurrence: occurrence.get_primary_and_pyname()[0].get_object() is not public, + ) + self.project.do(changes) + self.assertIn("direct = mod2.Target.Inner", self.mod3.read()) + self.assertIn("composed = Public.Inner", self.mod3.read()) + self.assertIn("direct_annotation: 'mod2.Target.Inner'", self.mod3.read()) + self.assertIn("composed_annotation: 'Public.Inner'", self.mod3.read()) + + def test_move_nested_class_filter_preserves_exact_proven_destination_sites(self): + for unknown in (False, True): + with self.subTest(unknown=unknown): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + pass + """)) + self.mod2.write(dedent("""\ + class Target: + class Inner: + pass + """)) + self.mod3.write(dedent("""\ + import mod1 + import mod2 + direct = mod1.Owner.Inner() + current = mod2.Target.Inner() + """) + (dedent("""\ + def unresolved(value): + return value.Inner + """) if unknown else "")) + current = self.project.get_pymodule(self.mod2)["Target"].get_object()["Inner"] + finder = occurrences.create_finder(self.project, "Inner", current) + proven = { + (site.resource.path, site.get_primary_range()) + for site in finder.find_occurrences(self.mod3) + } + self.assertEqual(1, len(proven)) + self.mod2.write(dedent("""\ + class Target: + pass + """)) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + destination_offset = self.mod2.read().index("Target") + with self.assertRaisesRegex(exceptions.RefactoringError, "Unproven"): + mover.get_changes(self.mod2, destination_offset=destination_offset) + def selected(site): + return (site.resource.path, site.get_primary_range()) not in proven + if unknown: + with self.assertRaisesRegex(exceptions.RefactoringError, "Unproven"): + mover.get_changes(self.mod2, destination_offset=destination_offset, match_filter=selected) + else: + self.project.do(mover.get_changes(self.mod2, destination_offset=destination_offset, match_filter=selected)) + self.assertNotIn("mod1.Owner.Inner", self.mod3.read()) + self.assertIn("current = mod2.Target.Inner()", self.mod3.read()) + namespace = runpy.run_path(self.mod2.real_path) + self.assertIsInstance(namespace["Target"].Inner(), namespace["Target"].Inner) + + def test_move_nested_class_rejects_import_collision(self): + self.mod1.write(dedent("""\ + from collections.abc import Mapping + class Owner: + class Inner: + value: Mapping + """)) + self.mod2.write(dedent("""\ + Mapping = str + class Target: + pass + """)) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + with self.assertRaisesRegex(exceptions.RefactoringError, "import conflicts"): + mover.get_changes(self.mod2, destination_offset=self.mod2.read().index("Target")) + + def test_move_nested_class_rewrites_quoted_type_operands_only(self): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + pass + """)) + self.mod2.write(dedent("""\ + class Target: + pass + """)) + self.mod4.write("class ForwardRef: pass\n") + self.mod3.write(dedent('''\ + from typing import Annotated as Meta, Literal as Value, ForwardRef as Ref + import typing as ty + from pkg.mod4 import ForwardRef as Fake + from mod1 import Owner as Legacy + text = "Legacy.Inner" + def consume(value: 'Legacy.Inner') -> "list['Legacy.Inner']": + """Legacy.Inner is explanatory text.""" + return [value] + variable: 'Legacy.Inner' + literal: Value['Legacy.Inner'] + metadata: Meta['Legacy.Inner', 'Legacy.Inner'] + explicit: Ref('Legacy.Inner') + qualified: ty.ForwardRef('Legacy.Inner') + unrelated: Fake('Legacy.Inner') + escaped: "list[\\\"Legacy.Inner\\\"]" + ''')) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + self.project.do(mover.get_changes(self.mod2, destination_offset=self.mod2.read().index("Target"))) + self.assertIn("value: 'mod2.Target.Inner'", self.mod3.read()) + self.assertIn('"list[\'mod2.Target.Inner\']"', self.mod3.read()) + self.assertIn("variable: 'mod2.Target.Inner'", self.mod3.read()) + self.assertIn("Value['Legacy.Inner']", self.mod3.read()) + self.assertIn("Meta['mod2.Target.Inner', 'Legacy.Inner']", self.mod3.read()) + self.assertIn("Ref('mod2.Target.Inner')", self.mod3.read()) + self.assertIn("ty.ForwardRef('mod2.Target.Inner')", self.mod3.read()) + self.assertIn("Fake('Legacy.Inner')", self.mod3.read()) + self.assertIn('escaped: "list[\\\"mod2.Target.Inner\\\"]"', self.mod3.read()) + self.assertIn('text = "Legacy.Inner"', self.mod3.read()) + self.assertIn('"""Legacy.Inner is explanatory text."""', self.mod3.read()) + + def test_move_nested_class_rejects_consumer_import_shadow(self): + self.mod1.write(dedent("""\ + class Owner: + class Inner: + pass + """)) + self.mod2.write(dedent("""\ + class Target: + pass + """)) + self.mod3.write(dedent("""\ + from mod1 import Owner + def use(mod2): + return Owner.Inner() + """)) + mover = move.create_move(self.project, self.mod1, self.mod1.read().index("Inner")) + with self.assertRaisesRegex(exceptions.RefactoringError, "consumer binding"): + mover.get_changes(self.mod2, destination_offset=self.mod2.read().index("Target")) + + def test_transfer_methods_to_enclosing_class_in_same_module(self): + self.mod1.write(dedent("""\ + value = 7 + + class Owner: + class Helpers: + @staticmethod + def make(number=2): + return value + number + + @classmethod + def name(cls): + return "name" + + result = Owner.Helpers.make() + """)) + self.mod2.write("from mod1 import Owner\nresult = Owner.Helpers.make(5)\n") + source = self.mod1.read() + changes = move.MoveMethod.get_class_changes( + self.project, + [ + (self.mod1, source.index("def make") + 4, "make"), + (self.mod1, source.index("def name") + 4, "name"), + ], + self.mod1, + source.index("class Owner") + 6, + resources=[self.mod1, self.mod2], + ) + self.project.do(changes) + changed = self.mod1.read() + self.assertNotIn("from mod1 import", changed) + namespace = {} + exec(changed, namespace) + owner = namespace["Owner"] + self.assertIsInstance(owner.__dict__["make"], staticmethod) + self.assertIsInstance(owner.__dict__["name"], classmethod) + self.assertEqual(9, namespace["result"]) + self.assertEqual(12, owner.make(5)) + self.assertEqual("name", owner.name()) + self.assertIn("Owner.make(5)", self.mod2.read()) + + def test_transfer_rejects_changed_class_scope_default(self): + self.mod1.write(dedent("""\ + class Owner: + value = 1 + class Helpers: + value = 2 + @staticmethod + def make(number=value): + return number + """)) + source = self.mod1.read() + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def make") + 4, "make")], + self.mod1, + source.index("class Owner") + 6, + resources=[self.mod1], + ) + self.assertEqual(source, self.mod1.read()) + + def test_transfer_rejects_changed_module_binding(self): + self.mod1.write(dedent("""\ + class Owner: + class Helpers: + @staticmethod + def make(): + return value + + value = Owner.Helpers.make + """)) + source = self.mod1.read() + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def make") + 4, "make")], + self.mod1, + source.index("class Owner") + 6, + resources=[self.mod1], + ) + self.assertEqual(source, self.mod1.read()) + + def test_transfer_rejects_ambiguous_module_reassignment(self): + self.mod1.write(dedent("""\ + value = 1 + class Owner: + class Helpers: + @staticmethod + def make(): + return value + + value = Owner.Helpers.make + """)) + source = self.mod1.read() + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def make") + 4, "make")], + self.mod1, + source.index("class Owner") + 6, + resources=[self.mod1], + ) + self.assertEqual(source, self.mod1.read()) + + def test_transfer_rejects_cross_module_redirected_globals(self): + for statement in ("return value", "value = 3\n return value"): + with self.subTest(statement=statement): + source = ( + "value = 1\nclass A:\n @staticmethod\n def f():\n" + " global value\n " + statement + "\n" + ) + destination = "value = 2\nclass B:\n pass\n" + self.mod1.write(source) + self.mod2.write(destination) + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def f") + 4, "f")], + self.mod2, + destination.index("class B") + 6, + resources=[self.mod1, self.mod2], + ) + self.assertEqual(source, self.mod1.read()) + self.assertEqual(destination, self.mod2.read()) + + def test_transfer_rejects_definition_time_changes(self): + declarations = [ + "@staticmethod\n def f(x=values[0]):", + "@staticmethod if values[0] else classmethod\n def f(x):", + ] + if sys.version_info < (3, 14): + # Python 3.14+ evaluates annotations lazily on access, in the same + # module globals before and after the move, so only eager + # annotations change with the definition time. + declarations.append("@staticmethod\n def f(x: values[0]):") + for declaration in declarations: + with self.subTest(declaration=declaration): + source = ( + "values = [1]\nclass A:\n " + declaration + + "\n return x\nvalues[0] = 2\nclass B:\n pass\n" + ) + self.mod1.write(source) + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def f") + 4, "f")], + self.mod1, + source.index("class B") + 6, + resources=[self.mod1], + ) + self.assertEqual(source, self.mod1.read()) + + @unittest.skipIf( + sys.version_info < (3, 14), "lazy function annotations require Python 3.14" + ) + def test_transfer_keeps_lazy_annotations(self): + source = ( + "values = [1]\nclass A:\n @staticmethod\n def f(x: values[0]):\n" + " return x\nvalues[0] = 2\nclass B:\n pass\n" + ) + self.mod1.write(source) + original = {} + exec(source, original) + self.project.do( + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def f") + 4, "f")], + self.mod1, + source.index("class B") + 6, + resources=[self.mod1], + ) + ) + moved = {} + exec(self.mod1.read(), moved) + self.assertEqual( + original["A"].f.__annotations__, moved["B"].f.__annotations__ + ) + + def test_transfer_rejects_early_same_module_references(self): + for early in ("result = A.f()", "class C:\n result = A.f()"): + with self.subTest(early=early): + source = ( + "class A:\n @staticmethod\n def f():\n return 1\n" + + early + "\nclass B:\n pass\n" + ) + self.mod1.write(source) + original = {} + exec(source, original) + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def f") + 4, "f")], + self.mod1, + source.index("class B") + 6, + resources=[self.mod1], + ) + self.assertEqual(source, self.mod1.read()) + + def test_transfer_rejects_descriptor_mutation_during_class_execution(self): + for statement in ( + "globals()['staticmethod'] = lambda f: 42", + "globals()['classmethod'] = lambda f: 42", + ): + with self.subTest(statement=statement): + descriptor = "classmethod" if "classmethod" in statement else "staticmethod" + source = ( + "class Owner:\n class Helpers:\n @" + descriptor + + "\n def f(x=None):\n return x\n " + + statement + "\n" + ) + self.mod1.write(source) + original = {} + exec(source, original) + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def f") + 4, "f")], + self.mod1, + source.index("class Owner") + 6, + resources=[self.mod1], + ) + self.assertEqual(source, self.mod1.read()) + + def test_transfer_rejects_destination_metaclass_effects(self): + for meta in ( + " @classmethod\n def __prepare__(cls, name, bases):\n" + " return {'staticmethod': lambda f: 42}\n", + " def __new__(cls, name, bases, namespace):\n" + " namespace['observed'] = 'f' in namespace\n" + " return super().__new__(cls, name, bases, namespace)\n", + ): + with self.subTest(meta=meta): + source = ( + "class Meta(type):\n" + meta + + "class Owner(metaclass=Meta):\n class Helpers:\n" + " @staticmethod\n def f():\n return 1\n" + ) + self.mod1.write(source) + original = {} + exec(source, original) + self.assertEqual(1, original["Owner"].Helpers.f()) + if "__new__" in meta: + self.assertFalse(original["Owner"].observed) + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def f") + 4, "f")], + self.mod1, + source.index("class Owner") + 6, + resources=[self.mod1], + ) + self.assertEqual(source, self.mod1.read()) + + def test_transfer_rejects_cross_module_descriptor_mutation(self): + for descriptor in ("staticmethod", "classmethod"): + with self.subTest(descriptor=descriptor): + source = ( + "class A:\n @" + descriptor + + "\n def f(x=None):\n return 1\n" + ) + destination = ( + "globals()['" + descriptor + "'] = lambda f: 42\n" + "class B:\n pass\n" + ) + self.mod1.write(source) + self.mod2.write(destination) + original = {} + exec(source, original) + self.assertEqual(1, original["A"].f()) + with self.assertRaises(exceptions.RefactoringError): + move.MoveMethod.get_class_changes( + self.project, + [(self.mod1, source.index("def f") + 4, "f")], + self.mod2, + destination.index("class B") + 6, + resources=[self.mod1, self.mod2], + ) + self.assertEqual(source, self.mod1.read()) + self.assertEqual(destination, self.mod2.read()) + def test_move_constant(self) -> None: self.origin_module.write("foo = 123\n") self._move(self.origin_module, self.origin_module.read().index("foo") + 1, self.destination_module) self.assertEqual("", self.origin_module.read()) self.assertEqual("foo = 123\n", self.destination_module.read()) + def test_move_constant_preserves_used_module_alias(self) -> None: + self.project.prefs["prefer_module_from_imports"] = True + self.mod4.write(dedent("""\ + def keep(): + return 1 + + value = 2 + """)) + self.mod5.write(dedent("""\ + def existing(): + return 3 + """)) + self.mod3.write(dedent("""\ + from pkg import mod5 + from pkg import mod4 as utils_module + + def run(): + return mod5.existing(), utils_module.keep(), utils_module.value + """)) + self._move(self.mod4, self.mod4.read().index("value"), self.mod5) + self.assertEqual( + dedent("""\ + from pkg import mod5 + from pkg import mod4 as utils_module + + def run(): + return mod5.existing(), utils_module.keep(), mod5.value + """), + self.mod3.read(), + ) + self.assertEqual(dedent("""\ + def keep(): + return 1 + + """), self.mod4.read()) + self.assertIn(dedent("""\ + value = 2 + """), self.mod5.read()) + def test_move_constant_2(self) -> None: self.origin_module.write("bar = 321\nfoo = 123\n") self._move(self.origin_module, self.origin_module.read().index("foo") + 1, self.destination_module) @@ -459,7 +1115,7 @@ class TestClass: self.origin_module.write(code) with self.assertRaisesRegex( exceptions.RefactoringError, - "Move refactoring should be performed on a global class, function or variable\.", + r"Move refactoring should be performed on a global class, function or variable\.", ): mover = move.create_move( self.project, self.origin_module, code.index("CONSTANT") + 1 @@ -472,7 +1128,7 @@ def a_func(): """)) with self.assertRaisesRegex( exceptions.RefactoringError, - "Moving global elements to the same module\.", + r"Moving global elements to the same module\.", ): self._move(self.origin_module, self.origin_module.read().index("a_func"), self.origin_module) @@ -563,6 +1219,55 @@ def a_func(): self.assertEqual(expected, self.destination_module.read()) self.assertEqual("", self.origin_module_in_pkg.read()) + def test_moving_global_replaces_relative_from_imports_in_consumers(self) -> None: + """A relative from-import of the moved name is rewritten like an absolute one""" + self.origin_module_in_pkg.write("class AClass:\n pass\n") + self.mod4.write("from .origin_module_in_pkg import AClass\na_var = AClass()\n") + self._move(self.origin_module_in_pkg, 6, self.destination_module_in_pkg) + self.assertEqual( + "import pkg.destination_module_in_pkg\n" + "a_var = pkg.destination_module_in_pkg.AClass()\n", + self.mod4.read(), + ) + + def test_moving_global_replaces_relative_package_imports_in_consumers(self) -> None: + """A relative import of the source module is rewritten to the destination""" + self.origin_module_in_pkg.write("class AClass:\n pass\n") + self.mod4.write( + "from . import origin_module_in_pkg\n" + "a_var = origin_module_in_pkg.AClass()\n" + ) + self._move(self.origin_module_in_pkg, 6, self.destination_module_in_pkg) + self.assertEqual( + "import pkg.destination_module_in_pkg\n" + "a_var = pkg.destination_module_in_pkg.AClass()\n", + self.mod4.read(), + ) + + def test_moving_global_retargets_package_init_import_and_alias(self) -> None: + package = testutils.create_package(self.project, "origin", self.mod4.parent) + origin = package.get_child("__init__.py") + origin.write("class Thing: pass\nother = 1\n") + self.mod4.write( + "from . import origin\n" + "from .origin import Thing as Alias, other\n" + "values = origin.Thing(), Alias(), other\n" + ) + self._move(origin, 6, self.destination_module_in_pkg) + self.assertNotIn("from . import origin", self.mod4.read()) + self.assertIn("from .origin import other", self.mod4.read()) + self.assertIn("from pkg.destination_module_in_pkg import Thing as Alias", self.mod4.read()) + self.assertIn("values = pkg.destination_module_in_pkg.Thing(), Alias(), other", self.mod4.read()) + + def test_moving_global_retargets_a_consumer_with_only_an_alias(self): + self.origin_module_in_pkg.write("value = 1\n") + self.mod4.write("from .origin_module_in_pkg import value as old\nresult = old\n") + self._move(self.origin_module_in_pkg, 0, self.destination_module_in_pkg) + self.assertEqual( + "from pkg.destination_module_in_pkg import value as old\nresult = old\n", + self.mod4.read(), + ) + def test_moving_modules_into_package(self) -> None: """Move global function where the destination module is in a package""" code = dedent("""\ @@ -1263,6 +1968,86 @@ def new_method(self): """) self.assertEqual(code, self.mod2.read()) + def _transfer_static_method(self, body: str) -> None: + self.origin_module.write( + dedent( + """\ + __all__ = ["Legacy"] + TOKEN = "source value" + + class Legacy: + @staticmethod + def execute(): + """ + ) + + " " + + body + + "\n" + ) + self.destination_module.write("class Target:\n pass\n") + changes = move.MoveMethod.get_class_changes( + self.project, + [ + ( + self.origin_module, + self.origin_module.read().index("execute"), + "execute", + ) + ], + self.destination_module, + self.destination_module.read().index("Target"), + ) + self.project.do(changes) + + def test_transfer_does_not_import_unused_export_metadata(self) -> None: + self._transfer_static_method("return TOKEN") + + self.assertIn("from origin_module import TOKEN", self.destination_module.read()) + self.assertNotIn("from origin_module import __all__", self.destination_module.read()) + self.assertIn('__all__ = ["Legacy"]', self.origin_module.read()) + + def test_transfer_imports_export_metadata_when_method_uses_it(self) -> None: + self._transfer_static_method("return __all__") + + self.assertIn("from origin_module import __all__", self.destination_module.read()) + + def test_transfer_preserves_future_annotations_without_origin_exports(self) -> None: + self.origin_module.write( + "from __future__ import annotations\n" + "__all__ = ['Legacy']\n" + "class Legacy:\n" + " @staticmethod\n" + " def execute(value: float) -> int:\n" + " return int(value)\n" + ) + self.destination_module.write("__all__ = ['Target']\nclass Target:\n pass\n") + changes = move.MoveMethod.get_class_changes( + self.project, + [(self.origin_module, self.origin_module.read().index("execute"), "execute")], + self.destination_module, + self.destination_module.read().index("Target", self.destination_module.read().index("class ")), + ) + self.project.do(changes) + target = runpy.run_path(self.destination_module.real_path)["Target"] + self.assertEqual(3, target.execute(3.7)) + self.assertEqual({"value": "float", "return": "int"}, target.execute.__annotations__) + self.assertNotIn("from origin_module import __all__", self.destination_module.read()) + + def test_moving_export_metadata_preserves_its_named_declarations(self) -> None: + self.origin_module.write("class Exported:\n pass\n__all__ = ['Exported']\n") + self._move( + self.origin_module, + self.origin_module.read().index("__all__"), + self.destination_module, + ) + source = self.project.get_pymodule(self.origin_module) + destination = self.project.get_pymodule(self.destination_module) + self.assertIs( + source.get_attribute("Exported").get_object(), + destination.get_attribute("Exported").get_object(), + ) + self.assertIn("__all__ = ['Exported']", self.destination_module.read()) + def test_moving_methods_getting_getting_changes_for_goal_class3(self) -> None: self.mod2.write(dedent("""\ class B(object): diff --git a/ropetest/refactor/namespace_package_rename_test.py b/ropetest/refactor/namespace_package_rename_test.py new file mode 100644 index 000000000..b6a819393 --- /dev/null +++ b/ropetest/refactor/namespace_package_rename_test.py @@ -0,0 +1,244 @@ +import subprocess +import sys +import unittest +from pathlib import Path +from textwrap import dedent + +from rope.refactor.rename import Rename +from ropetest import testutils + + +class NamespacePackageRenameTest(unittest.TestCase): + def setUp(self): + self.project = testutils.sample_project() + self.root = Path(self.project.address) + self.project.root.create_file("sentinel.txt").write("untouched\n") + + def tearDown(self): + testutils.remove_project(self.project) + + def _file(self, name, contents): + folder = self.project.root + parts = name.split("/") + for part in parts[:-1]: + if not folder.has_child(part): + folder.create_folder(part) + folder = folder.get_child(part) + resource = folder.create_file(parts[-1]) + resource.write(contents) + return resource + + def _run(self, module, expected): + result = subprocess.run( + [sys.executable, "-B", "-m", module], + cwd=self.root, + capture_output=True, + text=True, + timeout=15, + ) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(expected, result.stdout) + self.assertEqual("", result.stderr) + + def _rename(self, package, new_name, **options): + before = { + path.relative_to(self.root).as_posix(): path.read_bytes() + for path in self.root.rglob("*") + if path.is_file() and ".ropeproject" not in path.parts + } + changes = Rename(self.project, package).get_changes(new_name, **options) + after = { + path.relative_to(self.root).as_posix(): path.read_bytes() + for path in self.root.rglob("*") + if path.is_file() and ".ropeproject" not in path.parts + } + self.assertEqual(before, after) + self.project.do(changes) + self.assertEqual("untouched\n", (self.root / "sentinel.txt").read_text()) + + def test_namespace_directory_rename_preserves_saved_module_value(self): + foreign = dedent("""\ + class Foreign: + field = 'FOREIGN' + old = Foreign() + """) + subject = dedent("""\ + import valuespkg.foreign + class Other: + field = 'OTHER' + class Owner: + import valuespkg as slot + saved = Owner.slot + Owner.slot = Other() + print(saved.foreign.old.field) + """) + self._file("valuespkg/foreign.py", foreign) + self._file("subject.py", subject) + self._run("subject", "FOREIGN\n") + self._rename(self.project.root.get_child("valuespkg"), "renamed_pkg") + self.assertFalse((self.root / "valuespkg").exists()) + self.assertFalse((self.root / "renamed_pkg/__init__.py").exists()) + self.assertEqual(foreign, (self.root / "renamed_pkg/foreign.py").read_text()) + self.assertEqual( + subject.replace("import valuespkg", "import renamed_pkg"), + (self.root / "subject.py").read_text(), + ) + self._run("subject", "FOREIGN\n") + + def test_nested_namespace_directory_rename_preserves_relative_import(self): + self._file("outer/inner/value.py", "value = 42\n") + reader = dedent("""\ + from .value import value + print(value) + """) + self._file("outer/inner/reader.py", reader) + self._file( + "subject.py", + dedent("""\ + from outer.inner.value import value + print(value) + """), + ) + self._run("subject", "42\n") + self._run("outer.inner.reader", "42\n") + self._rename(self.project.root.get_child("outer/inner"), "renamed") + self.assertFalse((self.root / "outer/inner").exists()) + self.assertFalse((self.root / "outer/renamed/__init__.py").exists()) + self.assertEqual(reader, (self.root / "outer/renamed/reader.py").read_text()) + self.assertEqual( + dedent("""\ + from outer.renamed.value import value + print(value) + """), + (self.root / "subject.py").read_text(), + ) + self._run("subject", "42\n") + self._run("outer.renamed.reader", "42\n") + + def test_namespace_directory_rename_preserves_import_alias(self): + self._file("valuespkg/value.py", "value = 42\n") + self._file( + "subject.py", + dedent("""\ + from valuespkg import value as saved + print(saved.value) + """), + ) + self._run("subject", "42\n") + self._rename(self.project.root.get_child("valuespkg"), "renamed_pkg") + self.assertEqual( + dedent("""\ + from renamed_pkg import value as saved + print(saved.value) + """), + (self.root / "subject.py").read_text(), + ) + self.assertFalse((self.root / "valuespkg").exists()) + self._run("subject", "42\n") + + def test_empty_namespace_directory_rename_moves_directory(self): + package = self.project.root.create_folder("valuespkg") + self._file( + "subject.py", + dedent("""\ + import valuespkg + print(hasattr(valuespkg, '__path__')) + """), + ) + self._run("subject", "True\n") + self._rename(package, "renamed_pkg") + self.assertTrue((self.root / "renamed_pkg").is_dir()) + self.assertFalse((self.root / "valuespkg").exists()) + self.assertFalse((self.root / "renamed_pkg/__init__.py").exists()) + self._run("subject", "True\n") + + def test_namespace_directory_rename_with_only_importer_keeps_directory(self): + value = self._file("valuespkg/value.py", "value = 42\n") + subject = self._file("subject.py", "import valuespkg.value\n") + self._rename( + self.project.root.get_child("valuespkg"), + "renamed_pkg", + resources=[subject], + ) + self.assertEqual("import renamed_pkg.value\n", subject.read()) + self.assertTrue((self.root / "valuespkg").is_dir()) + self.assertFalse((self.root / "renamed_pkg").exists()) + self.assertEqual("value = 42\n", value.read()) + + def test_regular_package_directory_rename_preserves_initializer(self): + initializer = "value = 42\n" + self._file("valuespkg/__init__.py", initializer) + self._file( + "subject.py", + dedent("""\ + import valuespkg + print(valuespkg.value) + """), + ) + self._run("subject", "42\n") + self._rename(self.project.root.get_child("valuespkg"), "renamed_pkg") + self.assertFalse((self.root / "valuespkg").exists()) + self.assertEqual( + initializer, (self.root / "renamed_pkg/__init__.py").read_text() + ) + self._run("subject", "42\n") + + def test_namespace_directory_rename_with_all_package_files_moves_directory(self): + value = self._file("valuespkg/value.py", "value = 42\n") + other = self._file("valuespkg/other.py", "other = 7\n") + subject = self._file( + "subject.py", + dedent("""\ + from valuespkg.value import value + print(value) + """), + ) + self._run("subject", "42\n") + self._rename( + self.project.root.get_child("valuespkg"), + "renamed_pkg", + resources=[value, other, subject], + ) + self.assertFalse((self.root / "valuespkg").exists()) + self.assertEqual( + "other = 7\n", (self.root / "renamed_pkg/other.py").read_text() + ) + self._run("subject", "42\n") + + def test_namespace_directory_rename_with_partial_package_files_keeps_directory( + self, + ): + value = self._file("valuespkg/value.py", "value = 42\n") + other = self._file("valuespkg/other.py", "other = 7\n") + subject = self._file("subject.py", "import valuespkg.value\n") + self._rename( + self.project.root.get_child("valuespkg"), + "renamed_pkg", + resources=[value, subject], + ) + self.assertTrue((self.root / "valuespkg").is_dir()) + self.assertFalse((self.root / "renamed_pkg").exists()) + self.assertEqual("import renamed_pkg.value\n", subject.read()) + self.assertEqual("value = 42\n", value.read()) + self.assertEqual("other = 7\n", other.read()) + + def test_empty_namespace_directory_with_explicit_resources_keeps_directory(self): + package = self.project.root.create_folder("valuespkg") + subject = self._file("subject.py", "import valuespkg\n") + self._rename(package, "renamed_pkg", resources=[subject]) + self.assertTrue((self.root / "valuespkg").is_dir()) + self.assertFalse((self.root / "renamed_pkg").exists()) + self.assertEqual("import renamed_pkg\n", subject.read()) + + def test_regular_package_with_only_importer_keeps_initializer_and_directory(self): + initializer = self._file("valuespkg/__init__.py", "value = 42\n") + subject = self._file("subject.py", "import valuespkg\n") + self._rename( + self.project.root.get_child("valuespkg"), + "renamed_pkg", + resources=[subject], + ) + self.assertEqual("import renamed_pkg\n", subject.read()) + self.assertEqual("value = 42\n", initializer.read()) + self.assertTrue((self.root / "valuespkg").is_dir()) + self.assertFalse((self.root / "renamed_pkg").exists()) diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index f3320f450..96c72bb33 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -7,8 +7,6 @@ from rope.refactor import patchedast from ropetest import testutils -NameConstant = "Name" if sys.version_info <= (3, 8) else "NameConstant" - class PatchedASTTest(unittest.TestCase): def setUp(self): @@ -110,15 +108,15 @@ def test_bytes_string(self): checker = _ResultChecker(self, ast_frag) str_fragment = 'b"("' start = source.index(str_fragment) - checker.check_region("Bytes", start, start + len(str_fragment)) - checker.check_children("Bytes", [str_fragment]) + checker.check_region("Constant", start, start + len(str_fragment)) + checker.check_children("Constant", [str_fragment]) def test_integer_literals_and_region(self): source = "a = 10\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) start = source.index("10") - checker.check_region("Num", start, start + 2) + checker.check_region("Constant", start, start + 2) def test_negative_integer_literals_and_region(self): source = "a = -10\n" @@ -126,8 +124,8 @@ def test_negative_integer_literals_and_region(self): checker = _ResultChecker(self, ast_frag) start = source.index("-10") + 1 end = start + 2 - # Python 3 parses as UnaryOp(op=USub(), operand=Num(n=10)) - checker.check_region("Num", start, end) + # Python parses the sign as UnaryOp, with a Constant operand. + checker.check_region("Constant", start, end) def test_scientific_integer_literals_and_region(self): source = "a = -1.0e-3\n" @@ -135,15 +133,15 @@ def test_scientific_integer_literals_and_region(self): checker = _ResultChecker(self, ast_frag) start = source.index("-1.0e-3") + 1 end = start + 6 - # Python 3 parses as UnaryOp(op=USub(), operand=Num(n=10)) - checker.check_region("Num", start, end) + # Python parses the sign as UnaryOp, with a Constant operand. + checker.check_region("Constant", start, end) def test_hex_integer_literals_and_region(self): source = "a = 0x1\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) start = source.index("0x1") - checker.check_region("Num", start, start + 3) + checker.check_region("Constant", start, start + 3) def test_octal_integer_literals_and_region(self): source = "a = -0o1251\n" @@ -151,22 +149,58 @@ def test_octal_integer_literals_and_region(self): checker = _ResultChecker(self, ast_frag) start = source.index("-0o1251") + 1 end = start + 6 - # Python 3 parses as UnaryOp(op=USub(), operand=Num(n=10)) - checker.check_region("Num", start, end) - checker.check_children("Num", ["0o1251"]) + # Python parses the sign as UnaryOp, with a Constant operand. + checker.check_region("Constant", start, end) + checker.check_children("Constant", ["0o1251"]) def test_integer_literals_and_sorted_children(self): source = "a = 10\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Num", ["10"]) + checker.check_children("Constant", ["10"]) def test_ellipsis(self): source = "a[...]\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) start = source.index("...") - checker.check_region("Ellipsis", start, start + len("...")) + checker.check_region("Constant", start, start + len("...")) + + def test_positional_only_and_keyword_only_parameters(self): + source = "def f(a, /, b, *, c):\n pass\n" + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "arguments", + ["arg", "", ",", " ", "/", "", ",", " ", "arg", "", ",", " ", "*", + "", ",", " ", "arg"], + ) + + def test_bare_star_keyword_only_parameter_with_default(self): + source = "def f(*, key=None):\n pass\n" + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "arguments", ["*", "", ",", " ", "arg", "", "=", "", "Constant"] + ) + + def test_annotated_parameters_are_walked_as_nodes(self): + source = "def f(a: g(1), b: h(2)) -> None:\n pass\n" + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children("arguments", ["arg", "", ",", " ", "arg"]) + start = source.index("g(1)") + checker.check_region("Call", start, start + len("g(1)")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + def test_annotated_star_parameters(self): + source = "def f(*args: int, **kwargs: str):\n pass\n" + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "arguments", ["*", "", "arg", "", ",", " ", "**", "", "arg"] + ) + self.assertEqual(source, patchedast.write_ast(ast_frag)) def test_ass_name_node(self): source = "a = 10\n" @@ -181,7 +215,7 @@ def test_assign_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("Assign", 0, len(source) - 1) - checker.check_children("Assign", ["Name", " ", "=", " ", "Num"]) + checker.check_children("Assign", ["Name", " ", "=", " ", "Constant"]) @testutils.only_for_versions_higher("3.6") def test_ann_assign_node_without_target(self): @@ -198,7 +232,7 @@ def test_ann_assign_node_with_target(self): checker = _ResultChecker(self, ast_frag) checker.check_region("AnnAssign", 0, len(source) - 1) checker.check_children( - "AnnAssign", ["Name", "", ":", " ", "Name", " ", "=", " ", "Num"] + "AnnAssign", ["Name", "", ":", " ", "Name", " ", "=", " ", "Constant"] ) def test_add_node(self): @@ -206,21 +240,21 @@ def test_add_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["Num", " ", "+", " ", "Num"]) + checker.check_children("BinOp", ["Constant", " ", "+", " ", "Constant"]) def test_lshift_node(self): source = "1 << 2\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["Num", " ", "<<", " ", "Num"]) + checker.check_children("BinOp", ["Constant", " ", "<<", " ", "Constant"]) def test_and_node(self): source = "True and True\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BoolOp", 0, len(source) - 1) - checker.check_children("BoolOp", [NameConstant, " ", "and", " ", NameConstant]) + checker.check_children("BoolOp", ["Constant", " ", "and", " ", "Constant"]) def test_matmult_node(self): source = "a @ b\n" @@ -233,59 +267,91 @@ def test_basic_closing_parens(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["Num", " ", "+", " (", "Num", ")"]) + checker.check_children("BinOp", ["Constant", " ", "+", " (", "Constant", ")"]) def test_basic_opening_parens(self): source = "(1) + 2\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["(", "Num", ") ", "+", " ", "Num"]) + checker.check_children("BinOp", ["(", "Constant", ") ", "+", " ", "Constant"]) def test_basic_opening_biway(self): source = "(1) + (2)\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["(", "Num", ") ", "+", " (", "Num", ")"]) + checker.check_children("BinOp", ["(", "Constant", ") ", "+", " (", "Constant", ")"]) def test_basic_opening_double(self): source = "1 + ((2))\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["Num", " ", "+", " ((", "Num", "))"]) + checker.check_children("BinOp", ["Constant", " ", "+", " ((", "Constant", "))"]) def test_handling_comments(self): source = "(1 + #(\n2)\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("BinOp", ["Num", " ", "+", " #(\n", "Num"]) + checker.check_children("BinOp", ["Constant", " ", "+", " #(\n", "Constant"]) def test_handling_parens_with_spaces(self): source = "1 + (2\n )\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("BinOp", ["Num", " ", "+", " (", "Num", "\n )"]) + checker.check_children("BinOp", ["Constant", " ", "+", " (", "Constant", "\n )"]) def test_handling_strings(self): source = '1 + "("\n' ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("BinOp", ["Num", " ", "+", " ", "Str"]) + checker.check_children("BinOp", ["Constant", " ", "+", " ", "Constant"]) def test_handling_fstrings(self): source = '1 + f"("\n' ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("BinOp", ["Num", " ", "+", " ", "JoinedStr"]) + checker.check_children("BinOp", ["Constant", " ", "+", " ", "JoinedStr"]) + + def test_handling_hash_inside_implicitly_concatenated_fstrings(self): + # The walker searches the source as plain text. A `#` inside a + # string opened no comment, yet it made the rest of that line be + # skipped -- taking the closing paren with it -- and the walker + # then matched a paren belonging to some later statement. + source = dedent("""\ + assert flag, ( + f"one " + f"two (#123).") + """) + ast_frag = patchedast.get_patched_ast(source, True) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + def test_handling_hash_inside_a_plain_string(self): + source = dedent("""\ + assert flag, ( + "one " + "two (#123).") + """) + ast_frag = patchedast.get_patched_ast(source, True) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + def test_handling_paren_inside_a_triple_quoted_string(self): + source = dedent('''\ + SCHEMA = """ + PRIMARY KEY (a, b) + """ + run(SCHEMA) + ''') + ast_frag = patchedast.get_patched_ast(source, True) + self.assertEqual(source, patchedast.write_ast(ast_frag)) def test_handling_implicit_string_concatenation(self): source = "a = '1''2'" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Assign", ["Name", " ", "=", " ", "Str"]) - checker.check_children("Str", ["'1''2'"]) + checker.check_children("Assign", ["Name", " ", "=", " ", "Constant"]) + checker.check_children("Constant", ["'1''2'"]) def test_handling_implicit_string_concatenation_line_breaks(self): source = dedent("""\ @@ -293,15 +359,15 @@ def test_handling_implicit_string_concatenation_line_breaks(self): '2'""") ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Assign", ["Name", " ", "=", " ", "Str"]) - checker.check_children("Str", ["'1' \\\n'2'"]) + checker.check_children("Assign", ["Name", " ", "=", " ", "Constant"]) + checker.check_children("Constant", ["'1' \\\n'2'"]) def test_handling_explicit_string_concatenation_line_breaks(self): source = "a = ('1' \n'2')" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Assign", ["Name", " ", "=", " (", "Str", ")"]) - checker.check_children("Str", ["'1' \n'2'"]) + checker.check_children("Assign", ["Name", " ", "=", " (", "Constant", ")"]) + checker.check_children("Constant", ["'1' \n'2'"]) def test_not_concatenating_strings_on_separate_lines(self): source = dedent("""\ @@ -316,7 +382,7 @@ def test_handling_raw_strings(self): source = 'r"abc"\n' ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Str", ['r"abc"']) + checker.check_children("Constant", ['r"abc"']) @testutils.only_for_versions_higher("3.6") def test_handling_format_strings_basic(self): @@ -414,6 +480,78 @@ def test_handling_format_strings_with_implicit_join(self): ) checker.check_children("FormattedValue", ["{", "", "Name", "", "}"]) + @testutils.only_for_versions_higher("3.12") + def test_handling_format_strings_with_multiline_replacement_field(self): + source = 'x = (f"{a}{\n b\n}{c}").upper()\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index('f"') + checker.check_region("JoinedStr", start, source.index(").")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + @testutils.only_for_versions_higher("3.6") + def test_handling_format_strings_with_implicit_join_trailing_literal_part(self): + source = 'a = f"one {b}" f"two"\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "JoinedStr", ['f"', "one ", "FormattedValue", "", '"', ' f"two"'] + ) + checker.check_region("JoinedStr", 4, len('a = f"one {b}" f"two"')) + + @testutils.only_for_versions_higher("3.6") + def test_handling_format_strings_with_hash_in_joined_trailing_part(self): + source = dedent("""\ + E( + f"{a} one " + f"#52 two") + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "JoinedStr", ['f"', "", "FormattedValue", " one ", '"', '\n f"#52 two"'] + ) + checker.check_children("Call", ["Name", "", "(", "\n ", "JoinedStr", "", ")"]) + + @testutils.only_for_versions_higher("3.6") + def test_handling_format_strings_with_hash_in_joined_trailing_part_assignment(self): + source = 'a = 1\nb = (f"{a} one "\n f"#52 two")\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "JoinedStr", + ['f"', "", "FormattedValue", " one ", '"', '\n f"#52 two"'], + ) + + @testutils.only_for_versions_higher("3.6") + def test_handling_format_strings_followed_by_a_real_comment(self): + source = 'a = f"one {b}" # trailing comment\nc = 1\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children("JoinedStr", ['f"', "one ", "FormattedValue", "", '"']) + checker.check_region("JoinedStr", 4, len('a = f"one {b}"')) + def test_format_string_region_without_closing_quote_is_rejected(self): + source = 'f"abc{a}"\n' + node = ast.parse(source) + node.body[0].value.end_col_offset -= 1 + with self.assertRaisesRegex( + patchedast.MismatchedTokenError, "does not end with a quote delimiter" + ): + patchedast.patch_ast(node, source) + + def test_handling_format_strings_with_triple_quote_in_body(self): + # A double-quoted f-string whose body contains a ''' sequence must + # not confuse the end-quote detection (end_quote_char picked the + # longest quote found anywhere in the body instead of the matching + # delimiter at the end). + source = 'f"abc = \'\'\'{a}"\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "JoinedStr", ['f"', "abc = '''", "FormattedValue", "", '"'] + ) + checker.check_children("FormattedValue", ["{", "", "Name", "", "}"]) + @testutils.only_for_versions_higher("3.6") def test_handling_format_strings_with_format_spec(self): source = 'f"abc{a:01}"\n' @@ -443,12 +581,33 @@ def test_handling_format_strings_with_expression(self): checker.check_children("JoinedStr", ['f"', "abc", "FormattedValue", "", '"']) checker.check_children("FormattedValue", ["{", "", "BinOp", "", "}"]) + def test_escaped_format_fields_preserve_conditional_expression(self): + for prefix, literal_prefix in ( + ("", ""), + ("label = '\u03c0'\n", ""), + ("", "\u03c0"), + ): + with self.subTest(prefix=prefix, literal_prefix=literal_prefix): + source = prefix + dedent( + 'expression = f"${{frozen}}({{{literal}}})" if frozen else literal\n' + ) + source = source.replace('f"', f'f"{literal_prefix}', 1) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index("{literal}") + checker.check_region("FormattedValue", start, start + len("{literal}")) + checker.check_children( + "IfExp", + ["JoinedStr", " ", "if", " ", "Name", " ", "else", " ", "Name"], + ) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + def test_complex_number_literals(self): source = "1.0e2j + a" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("BinOp", ["Num", " ", "+", " ", "Name"]) - checker.check_children("Num", ["1.0e2j"]) + checker.check_children("BinOp", ["Constant", " ", "+", " ", "Name"]) + checker.check_children("Constant", ["1.0e2j"]) def test_ass_attr_node(self): source = "a.b = 1\n" @@ -485,7 +644,7 @@ def test_assert(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("Assert", 0, len(source) - 1) - checker.check_children("Assert", ["assert", " ", NameConstant]) + checker.check_children("Assert", ["assert", " ", "Constant"]) def test_assert2(self): source = 'assert True, "error"\n' @@ -493,7 +652,7 @@ def test_assert2(self): checker = _ResultChecker(self, ast_frag) checker.check_region("Assert", 0, len(source) - 1) checker.check_children( - "Assert", ["assert", " ", NameConstant, "", ",", " ", "Str"] + "Assert", ["assert", " ", "Constant", "", ",", " ", "Constant"] ) def test_aug_assign_node(self): @@ -501,20 +660,20 @@ def test_aug_assign_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("AugAssign", 0, len(source) - 1) - checker.check_children("AugAssign", ["Name", " ", "+", "", "=", " ", "Num"]) + checker.check_children("AugAssign", ["Name", " ", "+", "", "=", " ", "Constant"]) def test_bitand(self): source = "1 & 2\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["Num", " ", "&", " ", "Num"]) + checker.check_children("BinOp", ["Constant", " ", "&", " ", "Constant"]) def test_bitor(self): source = "1 | 2\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("BinOp", ["Num", " ", "|", " ", "Num"]) + checker.check_children("BinOp", ["Constant", " ", "|", " ", "Constant"]) def test_call_func(self): source = "f(1, 2)\n" @@ -522,7 +681,7 @@ def test_call_func(self): checker = _ResultChecker(self, ast_frag) checker.check_region("Call", 0, len(source) - 1) checker.check_children( - "Call", ["Name", "", "(", "", "Num", "", ",", " ", "Num", "", ")"] + "Call", ["Name", "", "(", "", "Constant", "", ",", " ", "Constant", "", ")"] ) def test_call_func_and_keywords(self): @@ -530,50 +689,24 @@ def test_call_func_and_keywords(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children( - "Call", ["Name", "", "(", "", "Num", "", ",", " ", "keyword", "", ")"] + "Call", ["Name", "", "(", "", "Constant", "", ",", " ", "keyword", "", ")"] ) - @testutils.only_for_versions_lower("3.5") def test_call_func_and_star_args(self): source = "f(1, *args)\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children( - "Call", ["Name", "", "(", "", "Num", "", ",", " ", "*", "", "Name", "", ")"] + "Call", ["Name", "", "(", "", "Constant", "", ",", " ", "Starred", "", ")"] ) - def test_call_func_and_star_argspython35(self): - source = "f(1, *args)\n" - ast_frag = patchedast.get_patched_ast(source, True) - checker = _ResultChecker(self, ast_frag) - checker.check_children( - "Call", ["Name", "", "(", "", "Num", "", ",", " ", "Starred", "", ")"] - ) - - @testutils.only_for_versions_lower("3.5") def test_call_func_and_only_dstar_args(self): - source = "f(**kwds)\n" - ast_frag = patchedast.get_patched_ast(source, True) - checker = _ResultChecker(self, ast_frag) - checker.check_children("Call", ["Name", "", "(", "", "**", "", "Name", "", ")"]) - - def test_call_func_and_only_dstar_args_python35(self): source = "f(**kwds)\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children("Call", ["Name", "", "(", "", "keyword", "", ")"]) - @testutils.only_for_versions_lower("3.5") def test_call_func_and_both_varargs_and_kwargs(self): - source = "f(*args, **kwds)\n" - ast_frag = patchedast.get_patched_ast(source, True) - checker = _ResultChecker(self, ast_frag) - checker.check_children( - "Call", - ["Name", "", "(", "", "*", "", "Name", "", ",", " ", "**", "", "Name", "", ")"], - ) - - def test_call_func_and_both_varargs_and_kwargs_python35(self): source = "f(*args, **kwds)\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) @@ -611,7 +744,7 @@ def test_simple_compare(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("Compare", 0, len(source) - 1) - checker.check_children("Compare", ["Num", " ", "<", " ", "Num"]) + checker.check_children("Compare", ["Constant", " ", "<", " ", "Constant"]) def test_multiple_compare(self): source = "1 < 2 <= 3\n" @@ -619,7 +752,7 @@ def test_multiple_compare(self): checker = _ResultChecker(self, ast_frag) checker.check_region("Compare", 0, len(source) - 1) checker.check_children( - "Compare", ["Num", " ", "<", " ", "Num", " ", "<=", " ", "Num"] + "Compare", ["Constant", " ", "<", " ", "Constant", " ", "<=", " ", "Constant"] ) def test_decorators_node(self): @@ -661,6 +794,27 @@ def f(*args, **kwds): "arguments", ["*", "", "args", "", ",", " ", "**", "", "kwds"] ) + def test_return_annotation_roundtrip(self): + source = "def f() -> Result[Value]:\n pass\n" + patched = patchedast.get_patched_ast(source, True) + self.assertEqual(source, patchedast.write_ast(patched)) + annotation = patched.body[0].returns + self.assertEqual( + "Result[Value]", + source[slice(*patchedast.node_region(annotation))], + ) + + def test_interleaved_class_header_roundtrip(self): + source = "class C(Base, metaclass=Meta, *bases, **options):\n pass\n" + patched = patchedast.get_patched_ast(source, True) + self.assertEqual(source, patchedast.write_ast(patched)) + for keyword in patched.body[0].keywords: + with self.subTest(keyword=keyword.arg): + self.assertEqual( + keyword.value.id, + source[slice(*patchedast.node_region(keyword.value))], + ) + def test_function_node(self): source = dedent("""\ def f(): @@ -688,6 +842,36 @@ async def f(): ["async", " ", "def", " ", "f", "", "(", "", "arguments", "", ")", "", ":", "\n ", "Pass"], ) + def test_argument_annotation(self): + source = "def f(items: list):\n pass\n" + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag.body[0].args.args[0]) + start = source.index("items") + end = source.index(")") + checker.check_region("arg", start, end) + start = source.index("list") + checker.check_region("Name", start, start + len("list")) + checker.check_children("arg", ["items", "", ":", " ", "Name"]) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + def test_argument_annotation_with_default(self): + source = dedent("""\ + async def f( + items : dict[str, list[int]] = None, # keep the annotation + fallback=None, + ): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + arguments = ast_frag.body[0].args + annotation = arguments.args[0].annotation + start = source.index("dict") + end = start + len("dict[str, list[int]]") + self.assertEqual((start, end), annotation.region) + self.assertEqual((source.index("items"), end), arguments.args[0].region) + self.assertEqual("None", source[slice(*arguments.defaults[0].region)]) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + def test_function_node2(self): source = dedent('''\ def f(p1, **p2): @@ -713,7 +897,7 @@ def test_dict_node(self): checker.check_region("Dict", 0, len(source) - 1) checker.check_children( "Dict", - ["{", "", "Num", "", ":", " ", "Num", "", ",", " ", "Num", "", ":", " ", "Num", "", "}"], + ["{", "", "Constant", "", ":", " ", "Constant", "", ",", " ", "Constant", "", ":", " ", "Constant", "", "}"], ) def test_dict_node_with_unpacking(self): @@ -730,7 +914,7 @@ def test_div_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("BinOp", 0, len(source) - 1) - checker.check_children("BinOp", ["Num", " ", "/", " ", "Num"]) + checker.check_children("BinOp", ["Constant", " ", "/", " ", "Constant"]) def test_for_node(self): source = dedent("""\ @@ -882,7 +1066,7 @@ def test_if_node(self): checker.check_region("If", 0, len(source) - 1) checker.check_children( "If", - ["if", " ", NameConstant, "", ":", "\n ", "Pass", "\n", "else", "", ":", "\n ", "Pass"], + ["if", " ", "Constant", "", ":", "\n ", "Pass", "\n", "else", "", ":", "\n ", "Pass"], ) def test_if_node2(self): @@ -896,7 +1080,7 @@ def test_if_node2(self): checker = _ResultChecker(self, ast_frag) checker.check_region("If", 0, len(source) - 1) checker.check_children( - "If", ["if", " ", NameConstant, "", ":", "\n ", "Pass", "\n", "If"] + "If", ["if", " ", "Constant", "", ":", "\n ", "Pass", "\n", "If"] ) def test_if_node3(self): @@ -912,7 +1096,7 @@ def test_if_node3(self): checker.check_region("If", 0, len(source) - 1) checker.check_children( "If", - ["if", " ", NameConstant, "", ":", "\n ", "Pass", "\n", "else", "", ":", "\n ", "If"], + ["if", " ", "Constant", "", ":", "\n ", "Pass", "\n", "else", "", ":", "\n ", "If"], ) def test_import_node(self): @@ -939,12 +1123,12 @@ def test_lambda_node(self): checker = _ResultChecker(self, ast_frag) checker.check_region("Lambda", 0, len(source) - 1) checker.check_children( - "Lambda", ["lambda", " ", "arguments", "", ":", " ", NameConstant] + "Lambda", ["lambda", " ", "arguments", "", ":", " ", "Constant"] ) expected_child = ast.arg.__name__ checker.check_children( "arguments", - [expected_child, "", ",", " ", expected_child, "", "=", "", "Num", "", ",", " ", "*", "", "z"], + [expected_child, "", ",", " ", expected_child, "", "=", "", "Constant", "", ",", " ", "*", "", "z"], ) def test_list_node(self): @@ -952,7 +1136,7 @@ def test_list_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("List", 0, len(source) - 1) - checker.check_children("List", ["[", "", "Num", "", ",", " ", "Num", "", "]"]) + checker.check_children("List", ["[", "", "Constant", "", ",", " ", "Constant", "", "]"]) def test_list_comp_node(self): source = "[i for i in range(1) if True]\n" @@ -964,7 +1148,7 @@ def test_list_comp_node(self): ) checker.check_children( "comprehension", - ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", NameConstant], + ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", "Constant"], ) def test_list_comp_node_with_multiple_comprehensions(self): @@ -978,7 +1162,7 @@ def test_list_comp_node_with_multiple_comprehensions(self): ) checker.check_children( "comprehension", - ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", NameConstant], + ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", "Constant"], ) def test_set_node(self): @@ -986,7 +1170,7 @@ def test_set_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_region("Set", 0, len(source) - 1) - checker.check_children("Set", ["{", "", "Num", "", ",", " ", "Num", "", "}"]) + checker.check_children("Set", ["{", "", "Constant", "", ",", " ", "Constant", "", "}"]) def test_set_comp_node(self): source = "{i for i in range(1) if True}\n" @@ -998,7 +1182,7 @@ def test_set_comp_node(self): ) checker.check_children( "comprehension", - ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", NameConstant], + ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", "Constant"], ) def test_dict_comp_node(self): @@ -1012,7 +1196,7 @@ def test_dict_comp_node(self): ) checker.check_children( "comprehension", - ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", NameConstant], + ["for", " ", "Name", " ", "in", " ", "Call", " ", "if", " ", "Constant"], ) def test_ext_slice_node(self): @@ -1021,7 +1205,7 @@ def test_ext_slice_node(self): checker = _ResultChecker(self, ast_frag) if sys.version_info >= (3, 9): checker.check_region("Tuple", 7, len(source) - 2) - checker.check_children("Tuple", ["Num", "", ",", "", "Slice"]) + checker.check_children("Tuple", ["Constant", "", ",", "", "Slice"]) else: checker.check_region("ExtSlice", 7, len(source) - 2) checker.check_children("ExtSlice", ["Index", "", ",", "", "Slice"]) @@ -1042,14 +1226,14 @@ def test_module_node(self): checker = _ResultChecker(self, ast_frag) checker.check_region("Module", 0, len(source)) checker.check_children("Module", ["", "Expr", "\n", "Pass", "\n"]) - checker.check_children("Str", ['"""docs"""']) + checker.check_children("Constant", ['"""docs"""']) def test_not_and_or_nodes(self): source = "not True or False\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children("Expr", ["BoolOp"]) - checker.check_children("BoolOp", ["UnaryOp", " ", "or", " ", NameConstant]) + checker.check_children("BoolOp", ["UnaryOp", " ", "or", " ", "Constant"]) def test_raise_node_bare(self): source = "raise\n" @@ -1079,7 +1263,7 @@ def f(): """) ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Return", ["return", " ", NameConstant]) + checker.check_children("Return", ["return", " ", "Constant"]) def test_empty_return_node(self): source = dedent("""\ @@ -1095,7 +1279,7 @@ def test_simple_slice_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children("Subscript", ["Name", "", "[", "", "Slice", "", "]"]) - checker.check_children("Slice", ["Num", "", ":", "", "Num"]) + checker.check_children("Slice", ["Constant", "", ":", "", "Constant"]) def test_slice_node2(self): source = "a[:]\n" @@ -1109,29 +1293,29 @@ def test_simple_subscript(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) if sys.version_info >= (3, 9): - checker.check_children("Subscript", ["Name", "", "[", "", "Num", "", "]"]) + checker.check_children("Subscript", ["Name", "", "[", "", "Constant", "", "]"]) else: checker.check_children("Subscript", ["Name", "", "[", "", "Index", "", "]"]) - checker.check_children("Index", ["Num"]) + checker.check_children("Index", ["Constant"]) def test_tuple_node(self): source = "(1, 2)\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Tuple", ["(", "", "Num", "", ",", " ", "Num", "", ")"]) + checker.check_children("Tuple", ["(", "", "Constant", "", ",", " ", "Constant", "", ")"]) def test_tuple_node2(self): source = "#(\n1, 2\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Tuple", ["Num", "", ",", " ", "Num"]) + checker.check_children("Tuple", ["Constant", "", ",", " ", "Constant"]) def test_tuple_with_complex_parentheses1(self): source = "a = ( # (he\n ((((), None))))\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children( - "Tuple", ["(", "", "Tuple", "", ",", " ", NameConstant, "", ")"] + "Tuple", ["(", "", "Tuple", "", ",", " ", "Constant", "", ")"] ) def test_tuple_with_complex_parentheses2(self): @@ -1139,7 +1323,7 @@ def test_tuple_with_complex_parentheses2(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children( - "Tuple", ["(", "", "((", "Str", "))", ",", " (", "Str", ")", "", ")"] + "Tuple", ["(", "", "((", "Constant", "))", ",", " (", "Constant", ")", "", ")"] ) def test_tuple_with_complex_parentheses3(self): @@ -1154,7 +1338,7 @@ def test_one_item_tuple_node(self): source = "(1,)\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Tuple", ["(", "", "Num", ",", ")"]) + checker.check_children("Tuple", ["(", "", "Constant", ",", ")"]) def test_empty_tuple_node(self): source = "()\n" @@ -1167,7 +1351,7 @@ def test_empty_tuple_node2(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children( - "Tuple", ["(", "", "Tuple", "", ",", " ", NameConstant, "", ")"] + "Tuple", ["(", "", "Tuple", "", ",", " ", "Constant", "", ")"] ) def test_empty_tuple_node3(self): @@ -1175,9 +1359,15 @@ def test_empty_tuple_node3(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children( - "Tuple", ["Tuple", "", ",", " ", NameConstant] + "Tuple", ["Tuple", "", ",", " ", "Constant"] ) + def test_empty_tuple_node4(self): + source = "a = (\n# foo,\n)\n" + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children("Tuple", ["(\n# foo,\n)"]) + def test_yield_node(self): source = dedent("""\ def f(): @@ -1185,7 +1375,7 @@ def f(): """) ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Yield", ["yield", " ", NameConstant]) + checker.check_children("Yield", ["yield", " ", "Constant"]) @testutils.only_for_versions_higher("3.3") def test_yield_from_node(self): @@ -1208,7 +1398,7 @@ def test_while_node(self): checker = _ResultChecker(self, ast_frag) checker.check_children( "While", - ["while", " ", NameConstant, "", ":", "\n ", "Pass", "\n", "else", "", ":", "\n ", "Pass"], + ["while", " ", "Constant", "", ":", "\n ", "Pass", "\n", "else", "", ":", "\n ", "Pass"], ) def test_with_node(self): @@ -1332,13 +1522,13 @@ def test_ignoring_comments(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) start = source.rindex("1") - checker.check_region("Num", start, start + 1) + checker.check_region("Constant", start, start + 1) def test_simple_sliceobj(self): source = "a[1::3]\n" ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) - checker.check_children("Slice", ["Num", "", ":", "", ":", "", "Num"]) + checker.check_children("Slice", ["Constant", "", ":", "", ":", "", "Constant"]) def test_ignoring_strings_that_start_with_a_char(self): source = 'r"""("""\n1\n' @@ -1355,7 +1545,7 @@ def test_if_exp_node(self): ast_frag = patchedast.get_patched_ast(source, True) checker = _ResultChecker(self, ast_frag) checker.check_children( - "IfExp", ["Num", " ", "if", " ", NameConstant, " ", "else", " ", "Num"] + "IfExp", ["Constant", " ", "if", " ", "Constant", " ", "else", " ", "Constant"] ) def test_delete_node(self): @@ -1364,36 +1554,6 @@ def test_delete_node(self): checker = _ResultChecker(self, ast_frag) checker.check_children("Delete", ["del", " ", "Name", "", ",", " ", "Name"]) - @testutils.only_for_versions_lower("3.5") - def test_starargs_before_keywords_legacy(self): - source = "foo(*args, a=1)\n" - ast_frag = patchedast.get_patched_ast(source, True) - checker = _ResultChecker(self, ast_frag) - checker.check_children( - "Call", - ["Name", "", "(", "", "*", "", "Name", "", ",", " ", "keyword", "", ")"], - ) - - @testutils.only_for_versions_lower("3.5") - def test_starargs_in_keywords_legacy(self): - source = "foo(a=1, *args, b=2)\n" - ast_frag = patchedast.get_patched_ast(source, True) - checker = _ResultChecker(self, ast_frag) - checker.check_children( - "Call", - ["Name", "", "(", "", "keyword", "", ",", " ", "*", "", "Name", "", ",", " ", "keyword", "", ")"], - ) - - @testutils.only_for_versions_lower("3.5") - def test_starargs_after_keywords_legacy(self): - source = "foo(a=1, *args)\n" - ast_frag = patchedast.get_patched_ast(source, True) - checker = _ResultChecker(self, ast_frag) - checker.check_children( - "Call", - ["Name", "", "(", "", "keyword", "", ",", " ", "*", "", "Name", "", ")"], - ) - def test_starargs_before_keywords(self): source = "foo(*args, a=1)\n" ast_frag = patchedast.get_patched_ast(source, True) @@ -1449,7 +1609,7 @@ def test_match_node_with_constant_match_value(self): checker = _ResultChecker(self, ast_frag) self.assert_single_case_match_block(checker, "MatchValue") checker.check_children("MatchValue", [ - "Num" + "Constant" ]) @testutils.only_for_versions_higher("3.10") @@ -1504,7 +1664,7 @@ def test_match_node_with_match_class(self): ")", ]) checker.check_children("MatchValue", [ - "Num" + "Constant" ]) @testutils.only_for_versions_higher("3.10") @@ -1597,6 +1757,35 @@ def test_match_node_with_match_sequence_with_no_parens(self): "MatchValue", "", ",", " ", "MatchValue", ]) + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_with_grouped_first_pattern(self): + source = dedent("""\ + match x: + case (("a" | None), None): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children("MatchSequence", [ + "(", "(", "MatchOr", ")", ",", " ", "MatchSingleton", "", ")", + ]) + start = source.index("((") + checker.check_region("MatchSequence", start, source.index(":\n", start)) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_of_parenthesized_patterns(self): + source = dedent("""\ + match x: + case (1), (2): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index("(1)") + checker.check_region("MatchSequence", start, start + len("(1), (2)")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + @testutils.only_for_versions_higher("3.10") def test_match_node_with_match_sequence_with_square_parens(self): source = dedent("""\ @@ -1843,7 +2032,7 @@ def test_match_node_with_match_mapping_match_as(self): checker.check_children("MatchMapping", [ "{", "", - "Str", + "Constant", "", ":", " ", @@ -2038,6 +2227,72 @@ def foo[**T = (A, B)](x): "Tuple", ]) + def test_string_literals_do_not_desynchronise_the_walker(self): + """Parentheses inside a triple-quoted string used to desynchronise + the source walker and surface a bare AttributeError. Tokens are no + longer matched inside string literals, so this module — reduced + from a real one (a SQL schema kept in a module-level string) — now + patches cleanly instead of failing. + """ + source = dedent("""\ + def check_transition(src: str, dst: str) -> None: + raise Refused( + f"{src} - {dst} isn't a declared transition. " + f"passing silently (#123).") + SCHEMA = \"\"\" + -- called `move() + PRIMARY KEY (source_site, item_id) + \"\"\" + class Store: + def __init__(self, path: str | Path) -> None: + self._conn.executescript(SCHEMA) + entry["identified_at"] = p + def get(self, source_site: str, item_id: str) -> Optional[dict]: + r + """) + + ast_frag = patchedast.get_patched_ast(source, True) + self.assertEqual(ast_frag.region, (0, len(source))) + + def test_consume_reports_mismatched_token_with_location(self): + """Whatever desynchronises the walker, the failure must surface as + rope's own error with a location, never a bare AttributeError from + deep inside the walker. + """ + source = patchedast._Source("def f(): pass") + with self.assertRaises(patchedast.MismatchedTokenError): + source.consume("class") + + + +class ConstantNodeTest(unittest.TestCase): + def test_parsed_literal_values_and_types(self): + for literal, expected, value_type in ( + ("None", None, type(None)), + ("True", True, bool), + ("False", False, bool), + ("1", 1, int), + ("1.5", 1.5, float), + ("2j", 2j, complex), + ('"hello"', "hello", str), + ('b"hello"', b"hello", bytes), + ("...", Ellipsis, type(Ellipsis)), + ): + with self.subTest(literal=literal): + source = dedent(f"""\ + value = {literal} + """) + ast_frag = patchedast.get_patched_ast(source, True) + node = ast_frag.body[0].value + self.assertIs(type(node), ast.Constant) + self.assertEqual(expected, node.value) + self.assertIs(type(node.value), value_type) + checker = _ResultChecker(self, ast_frag) + start = source.index(literal) + checker.check_region("Constant", start, start + len(literal)) + checker.check_children("Constant", [literal]) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + class _ResultChecker: def __init__(self, test_case, ast): @@ -2086,10 +2341,7 @@ class Search: def __call__(self, node): for text in goal: - if str(node).startswith(text): - self.result = node - break - if ast.get_node_type_name(node).startswith(text): + if type(node).__name__.startswith(text): self.result = node break return self.result is not None @@ -2098,6 +2350,12 @@ def __call__(self, node): ast.call_for_nodes(self.ast, search) return search.result + def check_region(self, text, start, end): + node = self._find_node(text) + if node is None: + self.test_case.fail("Node <%s> cannot be found" % text) + self.test_case.assertEqual((start, end), node.region) + def check_children(self, text, children): node = self._find_node(text) if node is None: @@ -2115,7 +2373,7 @@ def check_children(self, text, children): else: self.test_case.assertNotEqual("", text, "probably ignoring some node") self.test_case.assertTrue( - ast.get_node_type_name(child).startswith(expected), + type(child).__name__.startswith(expected), msg="Expected <%s> but was <%s>" - % (expected, ast.get_node_type_name(child)), + % (expected, type(child).__name__), ) diff --git a/ropetest/refactor/renametest.py b/ropetest/refactor/renametest.py index fb0270122..e577dfa9b 100644 --- a/ropetest/refactor/renametest.py +++ b/ropetest/refactor/renametest.py @@ -242,6 +242,131 @@ def test_renaming_inline_assignment(self): refactored, ) + @testutils.only_for_versions_higher("3.10") + def test_renaming_match_as_capture(self): + code = dedent("""\ + match value: + case [1, *_] as old_name: + print(old_name) + """) + refactored = self._local_rename( + code, code.rindex("old_name") + 1, "new_name" + ) + self.assertEqual( + dedent("""\ + match value: + case [1, *_] as new_name: + print(new_name) + """), + refactored, + ) + + @testutils.only_for_versions_higher("3.10") + def test_renaming_match_star_capture(self): + code = dedent("""\ + match value: + case [1, *old_name] if old_name: + print(old_name) + """) + refactored = self._local_rename( + code, code.index("old_name") + 1, "new_name" + ) + self.assertEqual( + dedent("""\ + match value: + case [1, *new_name] if new_name: + print(new_name) + """), + refactored, + ) + + @testutils.only_for_versions_higher("3.10") + def test_renaming_match_mapping_rest_capture(self): + code = dedent("""\ + match value: + case {"key": item, **old_name}: + print(old_name) + """) + refactored = self._local_rename( + code, code.index("old_name") + 1, "new_name" + ) + self.assertEqual( + dedent("""\ + match value: + case {"key": item, **new_name}: + print(new_name) + """), + refactored, + ) + + @testutils.only_for_versions_higher("3.12") + def test_renaming_type_alias(self): + code = dedent("""\ + type old_name = int + x: old_name = 1 + """) + refactored = self._local_rename(code, code.index("old_name") + 1, "new_name") + self.assertEqual( + dedent("""\ + type new_name = int + x: new_name = 1 + """), + refactored, + ) + + @testutils.only_for_versions_higher("3.12") + def test_renaming_type_alias_from_its_usage(self): + code = dedent("""\ + type old_name = int + def a_func(param: old_name) -> old_name: + pass + """) + offset = code.rindex("old_name") + 1 + refactored = self._local_rename(code, offset, "new_name") + self.assertEqual( + dedent("""\ + type new_name = int + def a_func(param: new_name) -> new_name: + pass + """), + refactored, + ) + + @testutils.only_for_versions_higher("3.12") + def test_renaming_type_alias_in_function_scope(self): + code = dedent("""\ + old_name = 1 + def a_func(): + type old_name = int + x: old_name = 1 + """) + offset = code.index("old_name", code.index("type")) + 1 + refactored = self._local_rename(code, offset, "new_name") + self.assertEqual( + dedent("""\ + old_name = 1 + def a_func(): + type new_name = int + x: new_name = 1 + """), + refactored, + ) + + @testutils.only_for_versions_higher("3.12") + def test_renaming_generic_type_alias(self): + code = dedent("""\ + type OldAlias[T] = list[T] + x: OldAlias[int] = [] + """) + refactored = self._local_rename(code, code.index("OldAlias") + 1, "NewAlias") + self.assertEqual( + dedent("""\ + type NewAlias[T] = list[T] + x: NewAlias[int] = [] + """), + refactored, + ) + def test_renaming_arguments_for_normal_args_changing_calls(self): code = dedent("""\ def a_func(p1=None, p2=None): @@ -319,6 +444,46 @@ def test_renaming_occurrence_in_f_string(self): refactored = self._local_rename(code, 2, "new_var") self.assertEqual(expected, refactored) + def test_renaming_preserves_normalized_fstring_identifiers(self): + code = dedent("""\ + target = 1 + K = 2 + unrelated = f'{\u212a}' + message = f'{target + \u212a}' + """) + expected = dedent("""\ + renamed = 1 + K = 2 + unrelated = f'{\u212a}' + message = f'{renamed + \u212a}' + """) + self.assertEqual( + expected, + self._local_rename(code, code.index("target"), "renamed"), + ) + + def test_renaming_multiline_fstring_preserves_continuation_indentation(self): + code = dedent('''\ + target = 1 + message = f"""{( + target + + target + + target + )}""" + ''') + expected = dedent('''\ + renamed = 1 + message = f"""{( + renamed + + renamed + + renamed + )}""" + ''') + self.assertEqual( + expected, + self._local_rename(code, code.index("target"), "renamed"), + ) + def test_renaming_occurrence_in_nested_f_string(self): code = dedent("""\ a_var = 20 @@ -331,6 +496,42 @@ def test_renaming_occurrence_in_nested_f_string(self): refactored = self._local_rename(code, 2, "new_var") self.assertEqual(expected, refactored) + def test_renaming_occurrence_in_f_string_with_reused_quotes(self): + code = dedent('''\ + a_var = {"key": 20} + a_string=f"{a_var['key']} and {a_var['key']:>2}" + ''') + expected = dedent('''\ + new_var = {"key": 20} + a_string=f"{new_var['key']} and {new_var['key']:>2}" + ''') + refactored = self._local_rename(code, 2, "new_var") + self.assertEqual(expected, refactored) + + def test_renaming_occurrence_in_f_string_format_spec(self): + code = dedent("""\ + a_var = 4 + a_string=f"{a_var:>{a_var}}" + """) + expected = dedent("""\ + new_var = 4 + a_string=f"{new_var:>{new_var}}" + """) + refactored = self._local_rename(code, 2, "new_var") + self.assertEqual(expected, refactored) + + def test_renaming_comprehension_binding_in_f_string_format_spec(self): + code = dedent("""\ + values = [1, 2] + result = [f"{item:>{item}}" for item in values] + """) + expected = dedent("""\ + values = [1, 2] + result = [f"{value:>{value}}" for value in values] + """) + refactored = self._local_rename(code, code.index("item in"), "value") + self.assertEqual(expected, refactored) + def test_renaming_attribute_occurrences_in_f_string(self): code = dedent("""\ class MyClass: @@ -640,6 +841,111 @@ def new_func(): mod2.read(), ) + def test_renaming_external_import_preserves_imported_name(self): + self.project.prefs.add("extension_modules", "builtins") + code = "from builtins import zip\nresult = list(zip([1], [2]))\n" + refactored = self._local_rename(code, code.rindex("zip"), "pairs") + self.assertEqual( + "from builtins import zip as pairs\nresult = list(pairs([1], [2]))\n", + refactored, + ) + namespace = {} + exec(refactored, namespace) + self.assertEqual([(1, 2)], namespace["result"]) + + def test_renaming_external_import_with_same_alias(self): + code = "from builtins import zip as zip\nresult = zip([], [])\n" + refactored = self._local_rename(code, code.rindex("zip"), "pairs") + self.assertEqual( + "from builtins import zip as pairs\nresult = pairs([], [])\n", + refactored, + ) + + def test_renaming_external_import_with_different_alias(self): + code = "from builtins import zip as pairs\nresult = pairs([], [])\n" + refactored = self._local_rename(code, code.rindex("pairs"), "new_pairs") + self.assertEqual( + "from builtins import zip as new_pairs\nresult = new_pairs([], [])\n", + refactored, + ) + + def test_renaming_multiline_external_import_after_unicode(self): + code = dedent("""\ + café = 1; from builtins import ( + zip as # Keep the imported name. + zip, + ) + result = zip([], []) + """) + refactored = self._local_rename(code, code.rindex("zip"), "pairs") + self.assertEqual( + dedent("""\ + café = 1; from builtins import ( + zip as # Keep the imported name. + pairs, + ) + result = pairs([], []) + """), + refactored, + ) + + def test_renaming_conditional_external_import_from_another_module(self): + compat = testutils.create_module(self.project, "compat") + compat.write(dedent("""\ + import sys + if sys.version_info[0] >= 3: + from builtins import map, zip + else: + zip = lambda *args: None + """)) + mod = testutils.create_module(self.project, "mod") + mod.write("from compat import zip\nresult = list(zip([1], [2]))\n") + self._rename(mod, mod.read().rindex("zip"), "pairs") + self.assertEqual( + dedent("""\ + import sys + if sys.version_info[0] >= 3: + from builtins import map, zip as pairs + else: + pairs = lambda *args: None + """), + compat.read(), + ) + self.assertEqual( + "from compat import pairs\nresult = list(pairs([1], [2]))\n", + mod.read(), + ) + + def test_renaming_import_from_external_python_module(self): + external_project = testutils.sample_project() + self.addCleanup(testutils.remove_project, external_project) + external = testutils.create_module(external_project, "external") + external.write("value = 42\n") + self.project.prefs.add("python_path", external_project.address) + code = "from external import value\nresult = value\n" + refactored = self._local_rename(code, code.rindex("value"), "new_value") + self.assertEqual( + "from external import value as new_value\nresult = new_value\n", + refactored, + ) + self.assertEqual("value = 42\n", external.read()) + + def test_renaming_external_import_does_not_resolve_unrelated_import(self): + invalid = testutils.create_module(self.project, "invalid") + invalid.write("def broken(:\n") + code = ( + "from invalid import other\n" + "from builtins import zip\n" + "result = zip([], [])\n" + ) + refactored = self._local_rename(code, code.rindex("zip"), "pairs") + self.assertEqual( + "from invalid import other\n" + "from builtins import zip as pairs\n" + "result = pairs([], [])\n", + refactored, + ) + def test_applying_all_changes_together(self): mod1 = testutils.create_module(self.project, "mod1") mod1.write(dedent("""\ diff --git a/ropetest/refactor/restructuretest.py b/ropetest/refactor/restructuretest.py index 9db6b77f7..9f1f71b9f 100644 --- a/ropetest/refactor/restructuretest.py +++ b/ropetest/refactor/restructuretest.py @@ -1,3 +1,4 @@ +import typing import unittest from textwrap import dedent @@ -28,6 +29,16 @@ def test_replacing_simple_patterns(self): self.project.do(refactoring.get_changes()) self.assertEqual("a = int(1)\nb = 1\n", self.mod.read()) + def test_replacing_argument_annotations(self): + source = "def f(items: list[list[int]] = None):\n return items\n" + self.mod.write(source) + refactoring = restructure.Restructure(self.project, "list", "tuple") + self.project.do(refactoring.get_changes()) + self.assertEqual( + "def f(items: tuple[tuple[int]] = None):\n return items\n", + self.mod.read(), + ) + def test_replacing_patterns_with_normal_names(self): refactoring = restructure.Restructure( self.project, "${a} = 1", "${a} = int(1)", args={"a": "exact"} @@ -182,6 +193,67 @@ def test_not_importing_names_when_there_are_no_changes(self): self.project.do(refactoring.get_changes()) self.assertEqual("a = True\n", self.mod.read()) + def test_match_filter_selects_restructured_matches(self): + self.mod.write("a = 1\nb = 1\n") + seen = [] + + def match_filter(pymodule, match): + seen.append(pymodule.get_resource()) + return match.get_region()[0] > 0 + + refactoring = restructure.Restructure( + self.project, "${x} = 1", "${x} = 2", match_filter=match_filter + ) + self.project.do(refactoring.get_changes()) + self.assertEqual("a = 1\nb = 2\n", self.mod.read()) + self.assertEqual([self.mod, self.mod], seen) + + def test_match_filter_rejecting_every_match_changes_nothing(self): + self.mod.write("a = 1\n") + refactoring = restructure.Restructure( + self.project, + "${x} = 1", + "${x} = 2", + imports=["import myconsts"], + match_filter=lambda pymodule, match: False, + ) + self.assertEqual([], refactoring.get_changes().changes) + self.assertEqual("a = 1\n", self.mod.read()) + + def test_hook_annotations_resolve_at_runtime(self): + hints = typing.get_type_hints(restructure.Restructure.__init__) + self.assertEqual( + typing.Optional[restructure.MatchFilter], hints["match_filter"] + ) + self.assertEqual( + typing.Optional[restructure.ImportFilter], hints["import_filter"] + ) + + def test_import_filter_selects_added_imports(self): + self.mod.write("a = 2\n") + received = [] + + def import_filter(resource, imports): + received.append((resource, imports)) + return [declaration for declaration in imports if "other" not in declaration] + + refactoring = restructure.Restructure( + self.project, + "${a} = 2", + "${a} = myconsts.two", + imports=["import myconsts", "import other"], + import_filter=import_filter, + ) + self.project.do(refactoring.get_changes()) + self.assertEqual( + dedent("""\ + import myconsts + a = myconsts.two + """), + self.mod.read(), + ) + self.assertEqual([(self.mod, ["import myconsts", "import other"])], received) + def test_handling_containing_matches(self): self.mod.write("a = 1 / 2 / 3\n") refactoring = restructure.Restructure( diff --git a/ropetest/refactor/signature_keyword_additiontest.py b/ropetest/refactor/signature_keyword_additiontest.py new file mode 100644 index 000000000..e5c9ee349 --- /dev/null +++ b/ropetest/refactor/signature_keyword_additiontest.py @@ -0,0 +1,1179 @@ +"""Public signature refactoring preserves modern Python source contracts.""" + +import inspect +import os +import subprocess +import sys +from textwrap import dedent + +import pytest + +from rope.base.exceptions import RefactoringError +from rope.base.project import Project +from rope.refactor.change_signature import ArgumentAdder, ChangeSignature + + +@pytest.mark.parametrize("foreign", [False, True]) +def test_sequence_hint_preserves_target_flow_to_unedited_consumer(tmp_path, foreign): + project = Project(str(tmp_path), ropefolder=None) + try: + dependency = project.root.create_file('signature_consumer.py') + dependency.write('def invoke(items):\n return items[0].run(7)\n') + resource = project.root.create_file('sample.py') + selected = 'Other' if foreign else 'Runner' + resource.write( + 'from collections.abc import Sequence\n' + 'from signature_consumer import invoke\n' + 'class Runner:\n def run(self, value: int):\n return value\n' + 'class Other:\n def run(self, value: int):\n return value + 4\n' + + f'def public_api(items: Sequence[{selected}]):\n return invoke(items)\n' + + f'print(public_api([{selected}()]))\n' + + 'answer = Runner().run(3)\n' + ) + before = resource.read() + original = subprocess.run([sys.executable, resource.real_path], capture_output=True, text=True, check=True) + assert original.stdout.strip() == ('11' if foreign else '7') + operation = ChangeSignature(project, resource, before.index('run')) + addition = ArgumentAdder(0, 'token', value="'received'", keyword_only=True) + if foreign: + project.do(operation.get_changes([addition], resources=[resource])) + actual = subprocess.run([sys.executable, resource.real_path], capture_output=True, text=True, check=True) + assert actual.stdout == original.stdout + else: + with pytest.raises(RefactoringError, match='Target namespace escapes'): + operation.get_changes([addition], resources=[resource]) + assert resource.read() == before + assert dependency.read() == 'def invoke(items):\n return items[0].run(7)\n' + finally: + project.close() + + +@pytest.mark.parametrize( + "source, entry", + [ + ( + "import importlib\nanswer = importlib.import_module('target').run(7)\n", + "consumer.answer", + ), + ( + "import importlib\ndef invoke():\n" + " return importlib.import_module('target').run(7)\nanswer = invoke()\n", + "consumer.answer", + ), + ( + "answer = __import__('target').__dict__['run'](7)\n", + "consumer.answer", + ), + ( + "def invoke(callback):\n return callback(__import__('target'))\n", + "consumer.invoke(lambda namespace: namespace.run(7))", + ), + ], +) +def test_required_keyword_retains_dynamic_consumers_outside_import_component( + tmp_path, source, entry +): + project = Project(str(tmp_path), ropefolder=None) + try: + target = project.root.create_file("target.py") + target.write("def run(value):\n return value\n") + consumer = project.root.create_file("consumer.py") + consumer.write(source) + executed = subprocess.run( + [sys.executable, "-c", f"import consumer; print({entry})"], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + assert executed.stdout.strip() == "7" + before = target.read(), consumer.read() + with pytest.raises(RefactoringError, match="Unresolved"): + ChangeSignature(project, target, target.read().index("run")).get_changes( + [ArgumentAdder(0, "token", value="'received'", keyword_only=True)], + resources=[target, consumer], + ) + assert (target.read(), consumer.read()) == before + finally: + project.close() + + +@pytest.mark.parametrize( + "header,call", + [ + ("def run(value: int, /, *, label: str = 'old') -> int:", "run(7, label='ok')"), + ("def run(value: int, **options: str) -> int:", "run(7, label='ok')"), + ("def run(value: int) -> int:", "run(\n 7, # keep this comment\n)"), + ("def run(value: int) -> int:", "int(f'{run(7)}')"), + ("def run(value: int, /, *, café: str = 'ação', **options: str) -> int:", "run(7, café='résumé')"), + ("def run(value: int, *rest: str, label: str = 'old') -> int:", "run(7, 'extra', label='ok')"), + ("def run(\n value: int, # retained parameter comment\n /,\n *,\n label: str = 'old',\n) -> int:", "run(7, label='ok')"), + ], +) +def test_required_keyword_preserves_existing_runtime_contract(tmp_path, header, call): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write(header + "\n return value\n\nanswer = " + call + "\n") + refactoring = ChangeSignature(project, resource, resource.read().index("run")) + addition = ArgumentAdder(0, "token", value="'received'", keyword_only=True, annotation="str") + project.do(refactoring.get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == 7 + signature = inspect.signature(namespace["run"]) + assert signature.parameters["token"].kind is inspect.Parameter.KEYWORD_ONLY + assert signature.parameters["token"].annotation is str + assert signature.parameters["value"].annotation is int + assert signature.return_annotation is int + with pytest.raises(TypeError, match="token"): + namespace["run"](7) + assert not ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.parametrize( + "value", ["'received'", "dict(label='received')['label']", "'re' + 'ceived'"] +) +def test_required_keyword_value_inside_fstring_is_written_legally(tmp_path, value): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "def run(value):\n return value, token\n\nanswer = f'{run(7)}'\n" + ) + addition = ArgumentAdder(0, "token", value=value, keyword_only=True) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == "(7, 'received')" + finally: + project.close() + + +def test_required_keyword_value_is_resolved_for_each_native_caller(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + dedent( + '''\ + def run(value: int) -> tuple[int, str]: + return value, token + + class Calls: + def direct(self): + dblink = "direct" + return run(7) + + def planned(self): + return run(8) + + def link(self): + return "planned" + + direct = Calls().direct() + planned = Calls().planned() + ''' + ) + ) + values = {"direct": "dblink", "planned": "self.link()"} + callers = [] + + def call_value(pymodule, occurrence): + scope = pymodule.get_scope().get_inner_scope_for_line(occurrence.lineno) + while scope.parent is not None: + name = scope.pyobject.get_name() + if name in values: + callers.append(name) + return values[name] + scope = scope.parent + raise AssertionError("No declared caller for keyword addition") + + addition = ArgumentAdder( + 0, "token", value=call_value, keyword_only=True, annotation="str" + ) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["direct"] == (7, "direct") + assert namespace["planned"] == (8, "planned") + assert sorted(callers) == ["direct", "planned"] + assert not ChangeSignature( + project, resource, resource.read().index("run") + ).get_changes([addition]).changes + finally: + project.close() + + +def test_ambiguous_keyword_expansion_refuses_the_unpublished_change(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(*, value: int) -> int:\n return value\n\nrun(**options)\n") + before = resource.read() + with pytest.raises(RefactoringError, match="expansion"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True, annotation="str") + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize( + "usage", + ["callback = run", "from functools import partial\ncallback = partial(run, 7)"], +) +def test_escaped_function_refuses_the_unpublished_change(tmp_path, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value: int) -> int:\n return value\n\n" + usage + "\n") + before = resource.read() + with pytest.raises(RefactoringError, match="non-call"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True, annotation="str") + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize("name", ["'run'", "selected_name"]) +@pytest.mark.parametrize("getter", ["getattr", "resolve"]) +def test_reflective_method_refuses_the_unpublished_change(tmp_path, name, getter): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + ("from builtins import getattr as resolve\n" if getter == "resolve" else "") + + "class Runner:\n @staticmethod\n def run(value: int) -> int:\n return value\n\n" + + "answer = " + getter + "(Runner, " + name + ")(7)\n" + ) + before = resource.read() + with pytest.raises(RefactoringError, match="signature consumer") as failure: + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True, annotation="str") + ]) + assert isinstance(failure.value.__cause__, RefactoringError) + assert ".py:" in str(failure.value) + assert resource.read() == before + finally: + project.close() + + +def test_decorated_method_and_foreign_binding_keep_their_runtime_contract(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Runner:\n @classmethod\n def run(cls, value: int, /, *, label: str = 'old') -> int:\n" + " return value\n\n" + "def foreign():\n def run(value: int) -> int:\n return value + 1\n return run(3)\n\n" + "answer = Runner.run(7)\nother = foreign()\n" + ) + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True, annotation="str") + ])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == 7 + assert namespace["other"] == 4 + assert inspect.signature(namespace["Runner"].run).parameters["token"].kind is inspect.Parameter.KEYWORD_ONLY + finally: + project.close() + + +def test_required_constructor_keyword_preserves_native_instantiation(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Builder:\n" + " def __init__(self, value: int):\n" + " self.value = value\n" + " self.token = token\n\n" + "answer = Builder(7)\n" + ) + addition = ArgumentAdder( + 0, "token", value="'received'", keyword_only=True, annotation="str" + ) + project.do( + ChangeSignature( + project, resource, resource.read().index("Builder") + ).get_changes([addition]) + ) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"].value == 7 + assert namespace["answer"].token == "received" + assert not ChangeSignature( + project, resource, resource.read().index("Builder") + ).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.parametrize( + "usage", + [ + "Factory = Builder\nanswer = Factory(7)", + "def create(factory):\n return factory(7)\nanswer = create(Builder)", + "class Child(Builder):\n pass\nanswer = Child(7)", + ], +) +def test_required_constructor_keyword_refuses_escaped_class(tmp_path, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Builder:\n def __init__(self, value):\n self.value = value\n\n" + + usage + + "\n" + ) + before = resource.read() + with pytest.raises(RefactoringError, match="non-call"): + ChangeSignature(project, resource, before.index("Builder")).get_changes([ + ArgumentAdder( + 0, "token", value="'received'", keyword_only=True, annotation="str" + ) + ]) + assert resource.read() == before + finally: + project.close() + + +def test_required_constructor_keyword_refuses_aliased_import(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Builder:\n def __init__(self, value):\n self.value = value\n" + ) + consumer = project.root.create_file("consumer.py") + consumer.write("from sample import Builder as Factory\nanswer = Factory(7)\n") + before = resource.read(), consumer.read() + with pytest.raises(RefactoringError, match="import alias"): + ChangeSignature( + project, resource, resource.read().index("Builder") + ).get_changes([ + ArgumentAdder( + 0, "token", value="'received'", keyword_only=True, annotation="str" + ) + ]) + assert (resource.read(), consumer.read()) == before + finally: + project.close() + + +@pytest.mark.parametrize( + "call", + [ + "run(value for value in produce())", + "run((value for value in produce()))", + "run(\n value for value in produce()\n # retained generator comment\n)", + ], +) +def test_required_keyword_preserves_generator_argument_evaluation(tmp_path, call): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "produced = []\n" + "def produce():\n produced.append('once')\n return (1, 2)\n\n" + "def run(values):\n return tuple(values), token\n\n" + "answer = " + call + "\n" + ) + addition = ArgumentAdder( + 0, "token", value="'received'", keyword_only=True, annotation="str" + ) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == ((1, 2), "received") + assert namespace["produced"] == ["once"] + if "# retained generator comment" in call: + assert "# retained generator comment" in resource.read() + assert not ChangeSignature( + project, resource, resource.read().index("run") + ).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +@pytest.mark.parametrize( + "header", + [ + "def run[T: (int, str)](value: T) -> T:", + "def run[\n T: (int, str), # retained bound comment\n](value: T) -> T:", + ], +) +def test_required_keyword_uses_parameter_region_after_type_bounds(tmp_path, header): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write(header + "\n return value\n\nanswer = run(7)\n") + addition = ArgumentAdder( + 0, "token", value="'received'", keyword_only=True, annotation="str" + ) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == 7 + assert namespace["run"].__type_params__[0].__constraints__ == (int, str) + assert inspect.signature(namespace["run"]).parameters["token"].kind is ( + inspect.Parameter.KEYWORD_ONLY + ) + assert not ChangeSignature( + project, resource, resource.read().index("run") + ).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.parametrize("decorator", ["decorate", "decorate()"]) +def test_required_keyword_refuses_unproven_decorator(tmp_path, decorator): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "def decorate(function=None):\n" + " def apply(function):\n" + " def wrapper(value):\n return function(value)\n" + " return wrapper\n" + " return apply if function is None else apply(function)\n\n" + f"@{decorator}\ndef run(value):\n return value\n\nanswer = run(7)\n" + ) + before = resource.read() + with pytest.raises(RefactoringError, match="decorator"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder( + 0, "token", value="'received'", keyword_only=True, annotation="str" + ) + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize("name", ["class", "await", "False"]) +def test_required_keyword_refuses_reserved_parameter_name(name): + with pytest.raises(RefactoringError, match="identifier"): + ArgumentAdder(0, name, value="'received'", keyword_only=True, annotation="str") + + +@pytest.mark.parametrize("name", ["'Builder'", "selected_name"]) +def test_required_constructor_keyword_refuses_reflective_factory(tmp_path, name): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Builder:\n def __init__(self, value):\n self.value = value\n" + ) + consumer = project.root.create_file("consumer.py") + consumer.write( + "import sample\nanswer = getattr(sample, " + name + ")(7)\n" + ) + before = resource.read(), consumer.read() + with pytest.raises(RefactoringError, match="signature consumer") as failure: + ChangeSignature( + project, resource, resource.read().index("Builder") + ).get_changes([ + ArgumentAdder( + 0, "token", value="'received'", keyword_only=True, annotation="str" + ) + ]) + assert isinstance(failure.value.__cause__, RefactoringError) + assert ".py:" in str(failure.value) + assert (resource.read(), consumer.read()) == before + finally: + project.close() + + +def test_optional_keyword_keeps_canonical_calls_and_escaped_references(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value: int) -> int:\n return value\n\ncallback = run\nanswer = callback(7)\n") + addition = ArgumentAdder(0, "environment", default="None", keyword_only=True, annotation="str | None") + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == 7 + assert inspect.signature(namespace["run"]).parameters["environment"].default is None + assert not ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition]).changes + finally: + project.close() + + +def test_optional_keyword_injects_only_declared_caller_override(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value: int) -> int:\n return value\n\ndef selected():\n return run(7)\n\ndef normal():\n return run(9)\n") + def value(module, occurrence): + scope = module.get_scope().get_inner_scope_for_line(occurrence.lineno) + return "'prepared'" if scope.pyobject.get_name() == "selected" else None + addition = ArgumentAdder(0, "environment", default="None", value=value, keyword_only=True, annotation="str | None") + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["selected"]() == 7 + assert namespace["normal"]() == 9 + assert not ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.parametrize( + "declarations", + [ + "class Builder:\n def __new__(cls, value):\n return super().__new__(cls)\n", + "class Base:\n def __new__(cls, value):\n return super().__new__(cls)\n\nclass Builder(Base):\n", + "class Meta(type):\n def __call__(cls, value):\n return super().__call__(value)\n\nclass Builder(metaclass=Meta):\n", + "class Meta(type):\n pass\n\nclass Base(metaclass=Meta):\n pass\n\nclass Builder(Base):\n", + "class Builder(UnknownBase):\n", + ], +) +def test_constructor_keyword_refuses_unproven_allocation(tmp_path, declarations): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + declarations + + " def __init__(self, value):\n self.value = value\n\nanswer = Builder(7)\n" + ) + before = resource.read() + with pytest.raises(RefactoringError, match="allocation"): + ChangeSignature( + project, resource, before.index("class Builder") + len("class ") + ).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize("default", [None, "1, 2"]) +@pytest.mark.parametrize("value", ["1, 2", "(1, 2)", "1, 2 # retained expression"]) +def test_keyword_tuple_expressions_preserve_grouping_and_idempotence(tmp_path, default, value): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value):\n return token\n\nanswer = run(7)\n") + addition = ArgumentAdder( + 0, "token", default=default, value=value, keyword_only=True, + annotation="tuple[int, int]", + ) + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == (1, 2) + assert inspect.signature(namespace["run"]).parameters["token"].annotation == tuple[int, int] + if default is not None: + assert namespace["run"](7) == (1, 2) + assert not ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.parametrize("field", ["value", "annotation", "default"]) +def test_keyword_expression_refuses_argument_injection(field): + with pytest.raises(RefactoringError, match="expression"): + ArgumentAdder( + 0, "token", keyword_only=True, + **{"value": "'received'", field: "'received', other='injected'"}, + ) + + +def test_keyword_callback_refuses_invalid_expression_without_publication(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value):\n return value\n\nanswer = run(7)\n") + before = resource.read() + with pytest.raises(RefactoringError, match="expression"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder( + 0, "token", value=lambda module, occurrence: "'received', other='injected'", + keyword_only=True, + ) + ]) + assert resource.read() == before + finally: + project.close() + + +def test_optional_constructor_override_refuses_class_decorator(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "def decorate(cls):\n def build(value):\n return cls(value)\n return build\n\n" + "@decorate\nclass Builder:\n def __init__(self, value):\n self.value = value\n\nanswer = Builder(7)\n" + ) + before = resource.read() + with pytest.raises(RefactoringError, match="decorator"): + ChangeSignature(project, resource, before.index("class Builder") + len("class ")).get_changes([ + ArgumentAdder(0, "token", default="None", value="'received'", keyword_only=True) + ]) + assert resource.read() == before + finally: + project.close() + + +def test_keyword_expression_refuses_invalid_context_before_publication(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value):\n return value\n\nanswer = run(7)\n") + before = resource.read() + with pytest.raises(RefactoringError, match="source context"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="(yield 1)", keyword_only=True) + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize( + "setup, usage", + [ + ("", 'globals()["run"](7)'), + ("", """int(f"{globals()['run'](7)}")"""), + ("", 'locals()["run"](7)'), + ("", 'vars()["run"](7)'), + ("from builtins import globals as namespace\n", 'namespace()["run"](7)'), + ('namespace = globals()\n', 'namespace["run"](7)'), + ('import builtins\n', 'getattr(builtins, "globals")()["run"](7)'), + ('import builtins\n', 'builtins.globals()["run"](7)'), + ('import builtins\n', 'builtins.getattr(builtins, "globals")()["run"](7)'), + ('import builtins\ndef factory():\n return builtins\n', 'getattr(factory(), "globals")()["run"](7)'), + ('import builtins\n', 'getattr(builtins, "glo" + "bals")()["run"](7)'), + ('import builtins\nnamespace = getattr(builtins, "globals")\n', 'namespace()["run"](7)'), + ('import builtins\n', 'builtins.__dict__["globals"]()["run"](7)'), + ('import builtins\n', 'getattr(builtins, "__dict__")["globals"]()["run"](7)'), + ], +) +def test_required_keyword_refuses_exposed_module_namespace(tmp_path, setup, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value):\n return value\n\n" + setup + "answer = " + usage + "\n") + before = resource.read() + namespace = {} + exec(before, namespace) + assert namespace["answer"] == 7 + with pytest.raises(RefactoringError, match="signature consumer") as failure: + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert isinstance(failure.value.__cause__, RefactoringError) + assert ".py:" in str(failure.value) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize( + "setup, usage", + [ + ("", 'Runner.__dict__["run"](Runner(), 7)'), + ("", 'vars(Runner)["run"](Runner(), 7)'), + ("", 'getattr(Runner, "__dict__")["run"](Runner(), 7)'), + ('namespace = Runner.__dict__\n', 'namespace["run"](Runner(), 7)'), + ], +) +def test_required_keyword_refuses_exposed_class_namespace(tmp_path, setup, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("class Runner:\n def run(self, value):\n return value\n\n" + setup + "answer = " + usage + "\n") + before = resource.read() + namespace = {} + exec(before, namespace) + assert namespace["answer"] == 7 + with pytest.raises(RefactoringError, match="signature consumer") as failure: + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert isinstance(failure.value.__cause__, RefactoringError) + assert ".py:" in str(failure.value) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize("callee", ["(run)", "((run))"]) +@pytest.mark.parametrize("default", [None, "'default'"]) +def test_parenthesized_native_call_receives_declared_keyword_override(tmp_path, callee, default): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value):\n return value, token\n\nanswer = " + callee + "(7)\n") + addition = ArgumentAdder(0, "token", default=default, value="'override'", keyword_only=True) + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == (7, "override") + assert not ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.parametrize( + "setup", + [ + "class Other:\n label = 'foreign'\nnamespace = Other.__dict__\n", + "def globals():\n return {'label': 'foreign'}\nnamespace = globals()\n", + "import operator\nclass Other:\n def label(self):\n return 'foreign'\nnamespace = {'label': operator.methodcaller('label')(Other())}\n", + "class methodcaller:\n def __init__(self, value):\n self.label = value\nnamespace = methodcaller('foreign').__dict__\n", + ], +) +def test_required_keyword_preserves_unrelated_namespace_bindings(tmp_path, setup): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(value):\n return value, token\n\n" + setup + "answer = run(7)\n") + addition = ArgumentAdder(0, "token", value="'received'", keyword_only=True) + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == (7, "received") + assert namespace["namespace"]["label"] == "foreign" + finally: + project.close() + + +@pytest.mark.parametrize( + "imports, usage", + [ + ("import operator\n", "operator.methodcaller('run', 7)(Runner())"), + ("from operator import methodcaller as invoke\n", "invoke('run', 7)(Runner())"), + ("import _operator\n", "_operator.methodcaller('run', 7)(Runner())"), + ("import operator\n", "operator.attrgetter('run')(Runner())(7)"), + ("from operator import attrgetter as resolve\n", "resolve('run')(Runner())(7)"), + ("import operator\nclass Holder:\n child = None\nholder = Holder()\nholder.child = Runner()\n", "operator.attrgetter('child.run')(holder)(7)"), + ("import operator\nselected_name = 'run'\n", "operator.methodcaller(selected_name, 7)(Runner())"), + ("import operator\n", "getattr(operator, 'methodcaller')('run', 7)(Runner())"), + ("import operator\n", "operator.__dict__['attrgetter']('run')(Runner())(7)"), + ], +) +def test_required_keyword_refuses_native_attribute_adapters(tmp_path, imports, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("class Runner:\n def run(self, value):\n return value\n\n" + imports + "answer = " + usage + "\n") + before = resource.read() + namespace = {} + exec(before, namespace) + assert namespace["answer"] == 7 + with pytest.raises(RefactoringError, match="signature consumer") as failure: + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert isinstance(failure.value.__cause__, RefactoringError) + assert ".py:" in str(failure.value) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize( + "imports, usage", + [ + ("from functools import partial\nfrom operator import methodcaller\n", "partial(methodcaller, 'run', 7)()(Runner())"), + ("from functools import partial\n", "partial(getattr, Runner(), 'run')()(7)"), + ("", "object.__getattribute__(Runner(), 'run')(7)"), + ("", "type.__getattribute__(Runner, 'run')(Runner(), 7)"), + ("import inspect\n", "dict(inspect.getmembers(Runner()))['run'](7)"), + ("from inspect import getmembers as members\n", "dict(members(Runner))['run'](Runner(), 7)"), + ], +) +def test_required_keyword_refuses_opaque_target_consumers(tmp_path, imports, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("class Runner:\n def run(self, value):\n return value\n\n" + imports + "answer = " + usage + "\n") + before = resource.read() + namespace = {} + exec(before, namespace) + assert namespace["answer"] == 7 + with pytest.raises(RefactoringError, match="opaque signature consumer"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize( + "imports, usage", + [ + ("", "object.__getattribute__(Other(), 'run')()"), + ("", "type.__getattribute__(Other, 'run')(Other())"), + ("import inspect\n", "dict(inspect.getmembers(Other()))['run']()"), + ], +) +def test_required_keyword_preserves_opaque_consumers_of_other_bindings(tmp_path, imports, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Runner:\n def run(self, value):\n return value, token\n\n" + "class Other:\n def run(self):\n return 'foreign'\n\n" + + imports + "foreign = " + usage + "\nanswer = Runner().run(7)\n" + ) + addition = ArgumentAdder(0, "token", value="'received'", keyword_only=True) + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == (7, "received") + assert namespace["foreign"] == "foreign" + finally: + project.close() + + +def test_required_keyword_preserves_analyzed_internal_owner_consumer(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Runner:\n def run(self, value):\n return value, token\n\n" + "def consume(owner):\n return owner.run(7)\n\nanswer = consume(Runner())\n" + ) + addition = ArgumentAdder(0, "token", value="'received'", keyword_only=True) + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == (7, "received") + assert not ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition]).changes + finally: + project.close() + + +@pytest.mark.parametrize( + "setup, usage", + [ + ("", 'eval("Runner().run(7)")'), + ("", 'exec("answer = Runner().run(7)")'), + ('source = "Runner().run(7)"\n', 'eval(source)'), + ('from builtins import eval as evaluate\n', 'evaluate("Runner().run(7)")'), + ], +) +def test_required_keyword_refuses_unproven_dynamic_code(tmp_path, setup, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("class Runner:\n def run(self, value):\n return value\n\n" + setup + "answer = " + usage + "\n") + before = resource.read() + namespace = {} + exec(before, namespace) + if not usage.startswith("exec("): + assert namespace["answer"] == 7 + with pytest.raises(RefactoringError, match="Dynamic code"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize( + "setup, collection", + [ + ("", "[Runner()]"), + ("", "(Runner(),)"), + ("", "{'one': Runner()}.values()"), + ('owners = [Runner()]\n', "owners"), + ('class Other:\n def run(self):\n return "foreign"\nowners = [Other(), Runner()]\n', "owners[1:]"), + ], +) +def test_required_keyword_refuses_nested_namespaces_in_opaque_collections(tmp_path, setup, collection): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "import inspect\nclass Runner:\n def run(self, value):\n return value\n\n" + + setup + "answer = dict(next(map(inspect.getmembers, " + collection + ")))['run'](7)\n" + ) + before = resource.read() + namespace = {} + exec(before, namespace) + assert namespace["answer"] == 7 + with pytest.raises(RefactoringError, match="opaque signature consumer"): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert resource.read() == before + finally: + project.close() + + +def test_required_keyword_preserves_literal_dynamic_code_and_foreign_collection(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "import inspect\nclass Runner:\n def run(self, value):\n return value, token\n\n" + "class Other:\n def run(self):\n return 'foreign'\n\n" + "exec('pass')\nliteral = eval('(1, 2)')\nowners = [Other()]\n" + "foreign = dict(next(map(inspect.getmembers, owners)))['run']()\nanswer = Runner().run(7)\n" + ) + addition = ArgumentAdder(0, "token", value="'received'", keyword_only=True) + project.do(ChangeSignature(project, resource, resource.read().index("run")).get_changes([addition])) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["answer"] == (7, "received") + assert namespace["foreign"] == "foreign" + assert namespace["literal"] == (1, 2) + finally: + project.close() + + +@pytest.mark.parametrize( + "setup, usage", + [ + ('enabled = True\nexecute = eval if enabled else str\n', "execute('Runner().run(7)')"), + ("", "__builtins__['eval']('Runner().run(7)')"), + ('from inspect import getmembers\nclass Holder:\n pass\nHolder.child = Runner()\n', "dict(getmembers(dict(getmembers(Holder()))['child']))['run'](7)"), + ], +) +def test_required_keyword_refuses_unproven_callable_or_transitive_namespace(tmp_path, setup, usage): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("class Runner:\n def run(self, value):\n return value\n\n" + setup + "answer = " + usage + "\n") + before = resource.read() + namespace = {} + exec(before, namespace) + assert namespace["answer"] == 7 + with pytest.raises(RefactoringError): + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, "token", value="'received'", keyword_only=True) + ]) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize("foreign,inert", [(False, True), (True, True), (False, False), (True, False)]) +@pytest.mark.parametrize("operation,statement", [ + ("__setitem__", "sink[0] = {value}"), + ("__setattr__", "sink.target = {value}"), + ("__iadd__", "sink += {value}"), + ("__add__", "sink += {value}"), + ("__radd__", "operand = {value}\noperand += sink"), + ("__add__", "sink + {value}"), + ("__radd__", "{value} + sink"), + ("__contains__", "{value} in sink"), + ("__contains__", "{value} not in sink"), + ("__eq__", "sink == {value}"), + ("__gt__", "{value} < sink"), +]) +def test_signature_effect_boundary_covers_external_store(tmp_path, foreign, inert, operation, statement): + root = tmp_path / 'project' + dependency = tmp_path / 'dependency' + root.mkdir() + dependency.mkdir() + (dependency / 'signature_sink.py').write_text( + "class Sink:\n def " + operation + "(self, " + + ("key, value" if operation in ("__setitem__", "__setattr__") else "value") + + "):\n object.__setattr__(self, 'result', value.run(7))\n return self\n", + encoding='utf-8', + ) + project = Project(str(root), ropefolder=None, python_path=[str(dependency)]) + try: + resource = project.root.create_file('sample.py') + value = 'Other()' if foreign else 'Runner()' + resource.write( + "from signature_sink import Sink\nclass Runner:\n def run(self, value):\n return value\n\n" + "class Other:\n def run(self, value):\n return " + + ("11" if inert else "value + 4") + "\n\n" + + "sink = Sink()\n" + + statement.format(value=value) + "\nprint(sink.result)\n" + + "answer = Runner().run(3)\n" + ) + before = resource.read() + environment = dict(os.environ, PYTHONPATH=str(dependency)) + original = subprocess.run([sys.executable, resource.real_path], env=environment, capture_output=True, text=True, check=True) + assert original.stdout.strip() == ('11' if foreign else '7') + refactoring = ChangeSignature(project, resource, before.index('run')) + addition = ArgumentAdder(0, 'token', value="'received'", keyword_only=True) + if foreign: + project.do(refactoring.get_changes([addition])) + actual = subprocess.run([sys.executable, resource.real_path], env=environment, capture_output=True, text=True, check=True) + assert actual.stdout.strip() == '11' + else: + with pytest.raises(RefactoringError, match="effect " + operation) as failure: + refactoring.get_changes([addition]) + assert "Target namespace escapes" in str(failure.value) + assert isinstance(failure.value.__cause__, RefactoringError) + assert "sample.py:" in str(failure.value) + assert resource.read() == before + finally: + project.close() + + +def test_signature_boundary_does_not_analyze_unimported_unknown_callback(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file('sample.py') + resource.write("def run(value):\n return value\n\nanswer = run(7)\n") + unrelated = project.root.create_file('unrelated.py') + unrelated.write("def invoke(callback):\n return callback()\n") + unrelated_before = unrelated.read() + addition = ArgumentAdder(0, 'token', value="'received'", keyword_only=True) + project.do(ChangeSignature(project, resource, resource.read().index('run')).get_changes([addition])) + namespace = {} + exec(resource.read(), namespace) + assert namespace['answer'] == 7 + assert unrelated.read() == unrelated_before + finally: + project.close() + + +@pytest.mark.parametrize("invoke", [False, True]) +def test_signature_boundary_refuses_opaque_callback_with_or_without_local_call(tmp_path, invoke): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "def run(value):\n return value\n" + "def invoke(callback):\n return callback('run(7)')\n" + "answer = run(7)\n" + + ("invoke(eval)\n" if invoke else "") + ) + before = resource.read() + namespace = {} + exec(before, namespace) + assert namespace["answer"] == 7 + assert namespace["invoke"](eval) == 7 + addition = ArgumentAdder(0, "token", value="'received'", keyword_only=True) + refactoring = ChangeSignature(project, resource, before.index("run")) + with pytest.raises(RefactoringError): + refactoring.get_changes([addition]) + assert resource.read() == before + current = {} + exec(resource.read(), current) + assert current["answer"] == 7 + assert current["invoke"](eval) == 7 + finally: + project.close() + + +@pytest.mark.parametrize("foreign", [False, True]) +@pytest.mark.parametrize("opaque", [False, True]) +def test_signature_boundary_covers_public_api_without_local_call(tmp_path, foreign, opaque): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Runner:\n def run(self, value):\n return value\n" + "class Other:\n def run(self, value):\n return " + + ("value + 4" if opaque else "11") + "\n" + "def public_api():\n return getattr(" + + ("Other" if foreign else "Runner") + + "(), 'run')(7)\n" + + ("def invoke(callback):\n return callback()\n" if opaque else "") + + "answer = Runner().run(3)\n" + ) + before = resource.read() + namespace = {} + exec(before, namespace) + expected = 11 if foreign else 7 + assert namespace["public_api"]() == expected + if opaque: + assert namespace["invoke"](lambda: namespace["Runner"]().run(7)) == 7 + addition = ArgumentAdder(0, "token", value="'received'", keyword_only=True) + refactoring = ChangeSignature(project, resource, before.index("run")) + if foreign and not opaque: + project.do(refactoring.get_changes([addition])) + namespace = {} + exec(resource.read(), namespace) + assert namespace["public_api"]() == expected + else: + with pytest.raises(RefactoringError) as failure: + refactoring.get_changes([addition]) + assert "sample.py:" in str(failure.value) + assert isinstance(failure.value.__cause__, RefactoringError) + assert resource.read() == before + finally: + project.close() + + +@pytest.mark.parametrize("arguments", ["", "'Runner().run(7)'"]) +def test_signature_boundary_refuses_public_callback_without_effect_contract(tmp_path, arguments): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "class Runner:\n def run(self, value):\n return value\n" + "def public_api(callback):\n return callback(" + arguments + ")\n" + "answer = Runner().run(3)\n" + ) + before = resource.read() + namespace = {} + exec(before, namespace) + callback = (lambda: namespace['Runner']().run(7)) if not arguments else lambda source: eval(source, namespace) + assert namespace['public_api'](callback) == 7 + with pytest.raises(RefactoringError, match="call") as failure: + ChangeSignature(project, resource, before.index("run")).get_changes([ + ArgumentAdder(0, 'token', value="'received'", keyword_only=True) + ]) + assert isinstance(failure.value.__cause__, RefactoringError) + assert 'sample.py:' in str(failure.value) + assert resource.read() == before + finally: + project.close() + + +def test_yield_from_does_not_hide_signature_before_nested_import(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "def earlier():\n yield from ()\n" + "def target(value):\n return value\n" + "def later():\n from math import floor\n return floor(1)\n" + ) + refactoring = ChangeSignature( + project, resource, resource.read().index("def target") + 4 + ) + changes = refactoring.get_changes( + [ArgumentAdder(0, "selected", default="None", keyword_only=True)] + ) + assert len(changes.changes) == 1 + project.do(changes) + namespace = {} + exec(compile(resource.read(), resource.path, "exec"), namespace) + assert namespace["target"](7) == 7 + parameter = inspect.signature(namespace["target"]).parameters["selected"] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY + assert parameter.default is None + assert namespace["later"]() == 1 + finally: + project.close() diff --git a/ropetest/refactor/signature_multiline_layouttest.py b/ropetest/refactor/signature_multiline_layouttest.py new file mode 100644 index 000000000..9e6187a1c --- /dev/null +++ b/ropetest/refactor/signature_multiline_layouttest.py @@ -0,0 +1,241 @@ +"""Public signature additions preserve multiline source and runtime binding.""" + +import inspect +from textwrap import dedent + +import pytest + +from rope.base.exceptions import RefactoringError +from rope.base.project import Project +from rope.refactor.change_signature import ArgumentAdder, ChangeSignature + + +@pytest.mark.parametrize( + "header, call, expected_header", + [ + ( + "def run(\n value: int,\n) -> int:", + "run(7)", + "def run(\n value: int,\n *,\n observer: object | None = None,\n) -> int:", + ), + ( + "def run(\n value: int, # retained Unicode café\n) -> int:", + "run(7)", + "def run(\n value: int, # retained Unicode café\n *,\n observer: object | None = None,\n) -> int:", + ), + ( + "def run(\n value: int # retained separator position\n) -> int:", + "run(7)", + "def run(\n value: int, # retained separator position\n *,\n observer: object | None = None,\n) -> int:", + ), + ( + "def run(\n value: int,\n **options: str\n) -> int:", + "run(7, label='same')", + "def run(\n value: int,\n *,\n observer: object | None = None,\n **options: str\n) -> int:", + ), + ], +) +def test_optional_keyword_retains_multiline_layout(tmp_path, header, call, expected_header): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + original = header + "\n return value\n\nanswer = " + call + "\n" + resource.write(original) + previous = {} + exec(compile(original, resource.path, "exec"), previous) + addition = ArgumentAdder( + 1, "observer", default="None", annotation="object | None", keyword_only=True + ) + project.do( + ChangeSignature(project, resource, original.index("run")).get_changes([addition]) + ) + changed = resource.read() + assert changed.startswith(expected_header + "\n") + assert changed.endswith("answer = " + call + "\n") + current = {} + exec(compile(changed, resource.path, "exec"), current) + assert current["answer"] == previous["answer"] == 7 + parameter = inspect.signature(current["run"]).parameters["observer"] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY + assert parameter.default is None + assert not ChangeSignature( + project, resource, changed.index("run") + ).get_changes([addition]).changes + finally: + project.close() + + +def test_required_keyword_retains_multiline_call_and_comment(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + dedent( + """\ + def run(value: int) -> int: + return value + + answer = run( + 7 # retained call comment + ) + """ + ) + ) + addition = ArgumentAdder(1, "token", value="'received'", keyword_only=True) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + changed = resource.read() + assert " 7, # retained call comment\n token='received',\n)" in changed + current = {} + exec(compile(changed, resource.path, "exec"), current) + assert current["answer"] == 7 + finally: + project.close() + + +def test_required_keyword_retains_multiline_generator_runtime(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "def run(values):\n return list(values)\n\n" + "answer = run(\n number for number in range(3)\n)\n" + ) + addition = ArgumentAdder(1, "token", value="'received'", keyword_only=True) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + current = {} + exec(compile(resource.read(), resource.path, "exec"), current) + assert current["answer"] == [0, 1, 2] + finally: + project.close() + + +def test_empty_multiline_header_uses_the_project_indentation(tmp_path): + project = Project(str(tmp_path), ropefolder=None, indent_size=2) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(\n):\n return len('value')\n\nanswer = run()\n") + addition = ArgumentAdder(0, "token", default="None", keyword_only=True) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + changed = resource.read() + assert changed.startswith("def run(\n *,\n token = None,\n):\n") + current = {} + exec(compile(changed, resource.path, "exec"), current) + assert current["answer"] == 5 + finally: + project.close() + + +def test_multiline_header_preserves_the_original_file_newlines(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write("def run(\r\n value: int,\r\n) -> int:\r\n return value\r\n") + addition = ArgumentAdder(1, "token", default="None", keyword_only=True) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + assert b"\n" not in resource.read_bytes().replace(b"\r\n", b"") + current = {} + exec(compile(resource.read(), resource.path, "exec"), current) + assert current["run"](7) == 7 + finally: + project.close() + + +def test_defaulted_addition_preserves_dynamic_import_calls_without_rewriting(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "import importlib\n" + "module_name = 'json'\n" + "decoder = importlib.import_module(module_name)\n" + "def run(\n value: int,\n) -> int:\n return value\n" + "answer = run(decoder.loads('7'))\n" + ) + original = {} + exec(resource.read(), original) + addition = ArgumentAdder(1, "token", default="None", keyword_only=True) + project.do( + ChangeSignature(project, resource, resource.read().index("run")).get_changes( + [addition] + ) + ) + current = {} + exec(resource.read(), current) + assert current["answer"] == original["answer"] == 7 + assert resource.read().endswith("answer = run(decoder.loads('7'))\n") + finally: + project.close() + + +@pytest.mark.parametrize( + "imports,decorator", + [ + ("from contextlib import contextmanager", "contextmanager"), + ("import contextlib", "contextlib.contextmanager"), + ("from contextlib import contextmanager as manage", "manage"), + ], +) +@pytest.mark.parametrize("default", [None, "None"]) +def test_native_contextmanager_forwards_signature_keywords(tmp_path, imports, decorator, default): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + imports + "\nclass Resource:\n @" + decorator + + "\n def open(\n self,\n value: int,\n ):\n" + " yield value\n\nwith Resource().open(7) as answer:\n observed = answer\n" + ) + previous = {} + exec(resource.read(), previous) + addition = ArgumentAdder( + 2, "token", default=default, + value="'received'" if default is None else None, + keyword_only=True, + ) + project.do( + ChangeSignature(project, resource, resource.read().index("open(")).get_changes( + [addition] + ) + ) + current = {} + exec(resource.read(), current) + assert current["observed"] == previous["observed"] == 7 + parameter = inspect.signature(current["Resource"]().open).parameters["token"] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY + assert parameter.default is (inspect.Parameter.empty if default is None else None) + finally: + project.close() + + +def test_contextmanager_homonym_does_not_gain_native_authority(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("sample.py") + resource.write( + "def contextmanager(function):\n return function\n" + "@contextmanager\ndef run(value):\n return value\n\nanswer = run(7)\n" + ) + previous = resource.read() + with pytest.raises(RefactoringError, match="decorator"): + ChangeSignature(project, resource, previous.index("run(")).get_changes( + [ArgumentAdder(1, "token", default="None", keyword_only=True)] + ) + assert resource.read() == previous + finally: + project.close() diff --git a/ropetest/refactor/similarfindertest.py b/ropetest/refactor/similarfindertest.py index b794a800b..222613f2f 100644 --- a/ropetest/refactor/similarfindertest.py +++ b/ropetest/refactor/similarfindertest.py @@ -54,6 +54,141 @@ def test_trivial_case(self): finder = self._create_finder("") self.assertEqual([], list(finder.get_match_regions("10"))) + def _assert_try_else_with_finally_regions(self, handler): + source = ( + "before = module.Model\n" + "try:\n value = 1\n" + + handler + + " ValueError:\n value = 2\n" + "else:\n value = module.Model\n" + "finally:\n value = 3\n" + ) + first = source.index("module.Model") + last = source.rindex("module.Model") + size = len("module.Model") + for warm_ast in (False, True): + with self.subTest(warm_ast=warm_ast): + finder = self._create_finder(source) + if warm_ast: + finder = similarfinder.SimilarFinder( + self.project.get_pymodule(self.mod) + ) + self.assertEqual( + [(first, first + size), (last, last + size)], + sorted(finder.get_match_regions("module.Model")), + ) + self.assertEqual( + [(first, first + size)], + list(finder.get_match_regions("module.Model", end=first + size)), + ) + + def test_matching_try_else_with_finally(self): + self._assert_try_else_with_finally_regions("except") + + @testutils.only_for("3.11") + def test_matching_try_star_else_with_finally(self): + self._assert_try_else_with_finally_regions("except*") + + def test_matching_argument_annotation(self): + source = "def foo(l: list):\n pass\n" + finder = similarfinder.RawSimilarFinder(source) + regions = [match.get_region() for match in finder.get_matches("${a}")] + start = source.index("list") + self.assertEqual([(start, start + len("list"))], regions) + + def test_matching_nested_argument_annotation(self): + source = "def foo(items: list[list[int]] = None):\n pass\n" + finder = self._create_finder(source) + regions = list(finder.get_match_regions("list")) + first = source.index("list") + second = source.index("list", first + 1) + self.assertEqual([(first, first + 4), (second, second + 4)], regions) + def test_return_annotation_names_have_match_regions(self): + source = "def f() -> target:\n return target\n" + finder = self._create_finder(source) + self.assertEqual( + [ + (source.rindex("target"), source.rindex("target") + len("target")), + (source.index("target"), source.index("target") + len("target")), + ], + list(finder.get_match_regions("target")), + ) + + def test_async_return_annotation_expression_has_match_region(self): + source = "async def f() -> list[target]:\n return []\n" + finder = self._create_finder(source) + start = source.index("list[target]") + self.assertEqual( + [(start, start + len("list[target]"))], + list(finder.get_match_regions("list[target]")), + ) + + def test_region_filter_can_pass_names_in_return_annotations(self): + source = "def f() -> target:\n return target\n" + finder = self._create_finder(source) + start = source.rindex("target") + end = start + len("target") + self.assertEqual( + [(start, end)], + list(finder.get_match_regions("target", start=start, end=end)), + ) + + def test_metaclass_only_header_names_have_match_regions(self): + source = "class C(metaclass=target):\n pass\n" + finder = self._create_finder(source) + start = source.index("target") + self.assertEqual( + [(start, start + len("target"))], + list(finder.get_match_regions("target")), + ) + + @testutils.only_for_versions_higher("3.12") + def test_type_parameter_bounds_have_match_regions(self): + source = "class Box[T: int]:\n pass\ntype Alias[U: int] = list[U]\n" + finder = self._create_finder(source) + first = source.index("int") + second = source.index("int", first + 1) + self.assertEqual( + [(first, first + len("int")), (second, second + len("int"))], + list(finder.get_match_regions("int")), + ) + + def test_fstring_literal_pieces_are_not_matched(self): + source = 'def f(x):\n return f"a{x}b"\n' + finder = self._create_finder(source) + self.assertEqual([], list(finder.get_match_regions('"a"'))) + self.assertEqual([], list(finder.get_match_regions('"b"'))) + + def test_fstring_format_specs_are_not_matched(self): + source = 'def f(x):\n return f"{x:%Y%m%d}"\n' + finder = self._create_finder(source) + self.assertEqual([], list(finder.get_match_regions('"%Y%m%d"'))) + + def test_fstring_replacement_fields_have_match_regions(self): + source = 'def f(p, w):\n return f"a{p.attr}b{p.when:{w}}"\n' + finder = self._create_finder(source) + start = source.index("p.attr") + self.assertEqual( + [(start, start + len("p.attr"))], + list(finder.get_match_regions("p.attr")), + ) + start = source.index("{w}") + 1 + self.assertEqual( + [(start, start + 1)], + list(finder.get_match_regions("w", start=start)), + ) + + def test_interleaved_class_header_names_have_match_regions(self): + source = "class C(Base, metaclass=Meta, *bases, **options):\n pass\n" + finder = self._create_finder(source) + for name in ("Base", "Meta", "bases", "options"): + with self.subTest(name=name): + start = source.index(name) + self.assertEqual( + [(start, start + len(name))], + list(finder.get_match_regions(name)), + ) + def test_constant_integer(self): source = "a = 10\n" finder = self._create_finder(source) @@ -144,6 +279,29 @@ def test_match_get_ast_for_statements(self): result = list(finder.get_matches("b = ${a}")) self.assertEqual("a", result[0].get_ast("a").id) + def test_expression_capture_region(self): + source = 'value = "é" + module.member\n' + finder = self._create_finder(source) + match = next(finder.get_matches('"é" + ${value}')) + start = source.index("module.member") + self.assertEqual( + (start, start + len("module.member")), match.get_capture_region("value") + ) + + def test_statement_capture_region(self): + source = "before = 0\nvalue = (first,\n second)\n" + finder = self._create_finder(source) + match = next(finder.get_matches("value = ${value}")) + start = source.index("(first") + end = source.index(")") + 1 + self.assertEqual((start, end), match.get_capture_region("value")) + + def test_missing_capture_region(self): + finder = self._create_finder("value = 1\n") + match = next(finder.get_matches("value = ${value}")) + with self.assertRaisesRegex(ValueError, "missing"): + match.get_capture_region("missing") + def test_matching_multiple_patterns(self): source = "c = a + b\n" finder = self._create_finder(source) diff --git a/ropetest/runmodtest.py b/ropetest/runmodtest.py index ae5234466..8cfdc67b7 100644 --- a/ropetest/runmodtest.py +++ b/ropetest/runmodtest.py @@ -22,11 +22,9 @@ def assert_collects_doa_data(project): @pytest.fixture def captured_run_module(project, tmp_path): def _run_module(file): - runner = project.pycore.run_module( - file, - stdout=stdout.open("w"), - ) - runner.wait_process() + with stdout.open("w") as output: + runner = project.pycore.run_module(file, stdout=output) + runner.wait_process() return stdout.read_text() stdout = tmp_path / "stdout" diff --git a/ropetest/simplifytest.py b/ropetest/simplifytest.py index a8795b988..f9d9a82b5 100644 --- a/ropetest/simplifytest.py +++ b/ropetest/simplifytest.py @@ -1,6 +1,7 @@ import unittest from rope.base import simplify +from ropetest import testutils class SimplifyTest(unittest.TestCase): @@ -62,3 +63,76 @@ def test_simplifying_f_string_containing_quotes(self): def test_simplifying_uppercase_f_string_containing_quotes(self): code = """s = Fr"..'{hello}'.."\n""" self.assertEqual("""s = Fr"..'{hello}'.."\n""", simplify.real_code(code)) + + # --- f-string literal/format-spec bracket folding (see CHANGELOG) --- + # An unmatched ``()[]{}`` in an f-string's *literal* or *format-spec* + # text must not be counted by the implicit-continuation pass, or every + # following newline is blanked and later offsets desync. Each of the + # following asserts the newline count is preserved (i.e. not folded). + + @testutils.only_for_versions_higher("3.12") + def test_fstring_literal_open_bracket_not_folded(self): + code = 'f"[{x}"\ndef g(): pass\n' + self.assertEqual(code.count("\n"), simplify.real_code(code).count("\n")) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_literal_open_paren_not_folded(self): + code = 'f"({n} items"\ndef g(): pass\n' + self.assertEqual(code.count("\n"), simplify.real_code(code).count("\n")) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_escaped_brace_not_folded(self): + code = 'f"{{"\ndef g(): pass\n' + self.assertEqual(code.count("\n"), simplify.real_code(code).count("\n")) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_format_spec_bracket_not_folded(self): + code = 'f"{x:>[}"\ndef g(): pass\n' + self.assertEqual(code.count("\n"), simplify.real_code(code).count("\n")) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_ansi_escape_not_folded(self): + code = 'f"\\x1b[K{x}"\ndef g(): pass\n' + self.assertEqual(code.count("\n"), simplify.real_code(code).count("\n")) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_multiline_bracket_not_folded(self): + code = 'f"""\n[{x}\n"""\ndef g(): pass\n' + self.assertEqual(code.count("\n"), simplify.real_code(code).count("\n")) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_nested_bracket_not_folded(self): + code = 'f"{f\'[{x}\'}"\ndef g(): pass\n' + self.assertEqual(code.count("\n"), simplify.real_code(code).count("\n")) + + # --- PRESERVED: real expression brackets inside an f-string must stay --- + # The fix must not over-blank real syntax. + + @testutils.only_for_versions_higher("3.12") + def test_fstring_real_list_brackets_preserved(self): + code = 'f"{[1,2][0]}"\ndef g(): pass\n' + self.assertEqual(code, simplify.real_code(code)) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_nested_string_literal_bracket_blanked(self): + # A bracket char that is literal data inside a nested string is not + # real syntax: it is blanked to a space (length preserved) so it + # cannot sway the continuation count. The structural brackets stay + # intact and the code below does not fold. + code = "f\"{d['}']}\"\ndef g(): pass\n" + expected = "f\"{d[' ']}\"\ndef g(): pass\n" + result = simplify.real_code(code) + self.assertEqual(expected, result) + self.assertEqual(len(code), len(result)) + + @testutils.only_for_versions_higher("3.12") + def test_fstring_format_spec_nested_expr_preserved(self): + code = 'f"{x:{w}}"\ndef g(): pass\n' + self.assertEqual(code, simplify.real_code(code)) + + def test_unterminated_fstring_falls_back_without_crashing(self): + # an invalid/incomplete f-string must not crash real_code or + # over-blank the code that follows it + code = 'f"[{x}\ndef g(): pass\n' + result = simplify.real_code(code) + self.assertIn("def g", result) diff --git a/ropetest/snapshot_resource_contracttest.py b/ropetest/snapshot_resource_contracttest.py new file mode 100644 index 000000000..3d38bcf1d --- /dev/null +++ b/ropetest/snapshot_resource_contracttest.py @@ -0,0 +1,566 @@ +"""Real source snapshots keep local and external resource reads closed.""" + +import ast +import os +import shutil +import sys +import typing +from textwrap import dedent, indent + +import pytest + +from rope.base import change, evaluate, pyobjects +from rope.base.fscommands import FileSystemCommands +from rope.base.project import NoProject, Project, get_no_project +from rope.refactor import patchedast, rename, similarfinder + + +@pytest.fixture +def alias_project(tmp_path): + project = Project(str(tmp_path)) + try: + yield project + finally: + project.close() + + +def test_installed_native_capture_region(alias_project): + source = 'prefix = "é"\nvalue = (first,\n second)\n' + resource = alias_project.root.create_file("captures.py") + resource.write(source) + module = alias_project.get_pymodule(resource) + match = next(similarfinder.SimilarFinder(module).get_matches("value = ${value}")) + start, end = match.get_capture_region("value") + assert source[start:end] == "(first,\n second)" + with pytest.raises(ValueError, match="missing"): + match.get_capture_region("missing") + + +def test_installed_mro_retains_unresolved_imported_base(alias_project): + provider = alias_project.root.create_file("provider.py") + provider.write("declared = 1\n") + resource = alias_project.root.create_file("consumer.py") + resource.write(dedent("""\ + from provider import missing + class Base(object): + member = 1 + class Derived(Base, missing): + pass + """)) + module = alias_project.get_pymodule(resource) + base = module["Base"].get_object() + derived = module["Derived"].get_object() + resolved, unresolved = derived.get_bases() + assert resolved is base + assert isinstance(unresolved, pyobjects.UnresolvedBase) + assert unresolved.pyobject is module["missing"].get_object() + assert derived.get_superclasses() == [base] + assert derived.get_mro() == [derived, *base.get_mro(), unresolved] + assert derived["member"] is base["member"] + + +def test_installed_class_header_retains_previous_import_binding(alias_project): + provider = alias_project.root.create_file("provider.py") + provider.write(dedent("""\ + class Foo: + member = 1 + """)) + resource = alias_project.root.create_file("consumer.py") + resource.write(dedent("""\ + from provider import Foo + class Foo(Foo): + pass + """)) + imported = alias_project.get_pymodule(provider)["Foo"].get_object() + derived = alias_project.get_pymodule(resource)["Foo"].get_object() + assert derived.get_mro() == [derived, imported] + assert derived["member"] is imported["member"] + + +def test_installed_class_subscription_retains_hook_result(alias_project): + resource = alias_project.root.create_file("subscription.py") + source = dedent("""\ + class Other: + pass + class Base: + def __class_getitem__(cls, item): + return Other + class Derived(Base[int]): + pass + """) + resource.write(source) + namespace = {} + exec(source, namespace) + assert namespace["Derived"].__bases__ == (namespace["Other"],) + module = alias_project.get_pymodule(resource) + assert module["Derived"].get_object().get_bases() == [ + module["Other"].get_object() + ] + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +def test_installed_pep695_bound_generic_preserves_ancestry(alias_project): + resource = alias_project.root.create_file("generic.py") + source = dedent("""\ + class Identity: + pass + class Ownership[Resource: Identity]: + pass + class Leaf(Ownership[Identity]): + pass + """) + resource.write(source) + namespace = {} + exec(source, namespace) + assert namespace["Ownership"] in namespace["Leaf"].__mro__ + assert namespace["Ownership"].__type_params__[0].__bound__ is namespace["Identity"] + module = alias_project.get_pymodule(resource) + assert module["Ownership"].get_object() in module["Leaf"].get_object().get_mro() + + +def test_installed_native_import_roots(alias_project): + source = ( + "import outer.child as overwritten, second\n" + "overwritten = None\n" + "from .relative import ignored\n" + "if False:\n from guarded.child import *\n" + "def nested():\n import nested_package.child\n" + ) + resource = alias_project.root.create_file("imports.py") + resource.write(source) + module = alias_project.get_pymodule(resource) + assert module.get_source_binding_index().absolute_import_roots() == frozenset( + {"outer", "second", "guarded", "nested_package"} + ) + + +def test_installed_native_assignment_string_sequence(alias_project): + resource = alias_project.root.create_file("metadata.py") + resource.write("shared = ('first',)\nclass Owner:\n names = tuple(shared)\n") + module = alias_project.get_pymodule(resource) + index = module.get_source_binding_index() + assert index.assignment_string_sequence("names") is None + scope = module["Owner"].get_object().get_scope() + assert index.assignment_string_sequence("names", scope) == ("first",) + assert index.assignment_string_sequence("names", class_metadata=True) == ("first",) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +@pytest.mark.parametrize("owner", ["module", "function", "class"]) +@pytest.mark.parametrize("bindings", ["alias", "ordinary-first", "alias-first", "repeated"]) +def test_type_alias_definition_location_preserves_first_binding(alias_project, owner, bindings): + declaration = "type Bag[T] = (\n SequenceOf[T]\n)\n" + body = "ordinary = 0\n\n" + if bindings == "ordinary-first": + body += "Bag = 1\n" + body += declaration + if bindings == "alias-first": + body += "Bag = 1\n" + elif bindings == "repeated": + body += "type Bag[U] = SequenceOf[U]\n" + body += "reference = Bag\n" + header = {"module": "", "function": "def factory():\n", "class": "class Owner:\n"}[owner] + source = "from collections.abc import Sequence as SequenceOf\n" + header + source += indent(body, " ") if header else body + resource = alias_project.root.create_file("declarations.py") + resource.write(source) + module = alias_project.get_pymodule(resource) + scope = module.get_scope() + if owner != "module": + scope = module["factory" if owner == "function" else "Owner"].get_object().get_scope() + alias = scope["Bag"] + first_line = source[:source.index("Bag")].count("\n") + 1 + ordinary_line = source[:source.index("ordinary =")].count("\n") + 1 + assert alias.get_definition_location() == (module, first_line) + assert scope["ordinary"].get_definition_location() == (module, ordinary_line) + if bindings in {"ordinary-first", "alias-first"}: + assert alias.get_object().get_type() is scope["ordinary"].get_object().get_type() + else: + assert alias.get_object() is pyobjects.get_unknown() + assert alias.get_definition_location() == (module, first_line) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +@pytest.mark.parametrize("owner", ["module", "function", "class"]) +def test_type_alias_location_does_not_evaluate_or_substitute_lazy_value(alias_project, owner): + source = "events = []\ndef resolve():\n events.append('value')\n raise RuntimeError('lazy value')\n" + declaration = "type Bag[T] = resolve()\n" + if owner == "module": + source += declaration + "selected = Bag\n" + elif owner == "class": + source += "class Owner:\n" + indent(declaration, " ") + "selected = Owner.Bag\n" + else: + source += "def factory():\n" + indent(declaration + "return Bag\n", " ") + "selected = factory()\n" + resource = alias_project.root.create_file("lazy.py") + resource.write(source) + module = alias_project.get_pymodule(resource) + scope = module.get_scope() + if owner != "module": + scope = module["factory" if owner == "function" else "Owner"].get_object().get_scope() + alias = scope["Bag"] + assert alias.get_definition_location() == (module, source[:source.index("type Bag")].count("\n") + 1) + assert alias.get_object() is pyobjects.get_unknown() + namespace = {} + exec(compile(source, resource.real_path, "exec"), namespace) + assert isinstance(namespace["selected"], typing.TypeAliasType) + assert namespace["events"] == [] + with pytest.raises(RuntimeError, match="lazy value"): + namespace["selected"].__value__ + assert namespace["events"] == ["value"] + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +@pytest.mark.parametrize("method", ["__init__", "method"]) +def test_method_type_alias_is_local_not_an_instance_attribute(alias_project, method): + source = f"class Owner:\n ordinary = 0\n def {method}(self):\n type Bag[T] = list[T]\n value = 1\n self.actual = value\n" + resource = alias_project.root.create_file("methods.py") + resource.write(source) + module = alias_project.get_pymodule(resource) + owner = module["Owner"].get_object() + assert "Bag" not in owner.get_attributes() + assert "value" not in owner.get_attributes() + assert owner["ordinary"].get_definition_location() == (module, 2) + assert owner["actual"].get_definition_location() == (module, 6) + scope = owner[method].get_object().get_scope() + assert scope["Bag"].get_definition_location() == (module, 4) + assert scope["value"].get_definition_location() == (module, 5) + assert scope["Bag"].get_object() is pyobjects.get_unknown() + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +@pytest.mark.parametrize("imports,reference", [ + ("from supplier import Bag", "Bag"), + ("from supplier import Bag as Alias", "Alias"), + ("import supplier", "supplier.Bag"), +]) +def test_imported_type_alias_location_and_native_rename(alias_project, imports, reference): + supplier = alias_project.root.create_file("supplier.py") + source = "from collections.abc import Sequence\n\ntype Bag[T] = Sequence[T]\n" + supplier.write(source) + client = alias_project.root.create_file("client.py") + client_source = imports + "\nreference = " + reference + "[int]\n" + client.write(client_source) + producer = alias_project.get_pymodule(supplier) + consumer = alias_project.get_pymodule(client) + offset = client_source.rindex(reference.rsplit(".", 1)[-1]) + 1 + binding = evaluate.eval_location(consumer, offset) + assert binding.get_definition_location() == (producer, 3) + alias_project.do(rename.Rename(alias_project, supplier, source.index("Bag")).get_changes("Crate")) + assert supplier.read() == source.replace("Bag", "Crate") + assert client.read() == client_source.replace("Bag", "Crate") + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +@pytest.mark.parametrize("forward", [False, True]) +@pytest.mark.parametrize("shadowed", [False, True]) +def test_class_type_alias_value_resolves_its_sibling(alias_project, forward, shadowed): + declaration = "type Bag[T] = list[T]\n" + reference = "type Items = Bag[int]\n" + body = reference + declaration if forward else declaration + reference + source = "type Bag[T] = tuple[T, ...]\n" if shadowed else "" + source += "class Owner:\n" + indent(body, " ") + resource = alias_project.root.create_file("class_aliases.py") + resource.write(source) + module = alias_project.get_pymodule(resource) + owner = module["Owner"].get_object() + + binding = evaluate.eval_location(module, source.index("Bag[int]") + 1) + + assert binding is owner["Bag"] + namespace = {} + exec(compile(source, resource.real_path, "exec"), namespace) + runtime_owner = namespace["Owner"] + assert typing.get_origin(runtime_owner.Items.__value__) is runtime_owner.Bag + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="PEP 695 requires Python 3.12") +def test_conditional_class_type_alias_retains_ambiguous_binding(alias_project): + source = ( + "condition = True\n" + "class Owner:\n" + " if condition:\n" + " type Bag[T] = list[T]\n" + " type Items = Bag[int]\n" + ) + resource = alias_project.root.create_file("conditional_alias.py") + resource.write(source) + module = alias_project.get_pymodule(resource) + binding = evaluate.eval_location(module, source.index("Bag[int]") + 1) + + assert binding.is_refactoring_identity_safe is False + assert binding.may_reference(module["Owner"].get_object()["Bag"]) + + +def test_default_project_initialization_has_no_warnings(tmp_path, recwarn): + project = Project(str(tmp_path)) + try: + assert project.root.pathlib == tmp_path + finally: + project.close() + assert not recwarn.list + + +def test_configured_source_folder_selection_has_no_warnings(tmp_path, recwarn): + source = tmp_path / "src" + source.mkdir() + project = Project(str(tmp_path), source_folders=["src"]) + try: + assert project.get_resource("src") in project.get_source_folders() + finally: + project.close() + assert not recwarn.list + + +@pytest.mark.parametrize("handler", ["except", "except*"]) +def test_try_else_finally_expression_regions_and_matching(tmp_path, handler): + if handler == "except*" and sys.version_info < (3, 11): + pytest.skip("exception groups require Python 3.11") + source = dedent(f"""\ + def execute(self): + try: + self.run() + {handler} Exception: + self.failed() + else: + ( + self._record_execution_metrics + ) + finally: + self.cleanup() + """) + tree = patchedast.get_patched_ast(source, True) + expression = tree.body[0].body[0].orelse[0].value + start = source.index("self._record_execution_metrics") + expected = (start, start + len("self._record_execution_metrics")) + assert expression.region == expected + assert patchedast.write_ast(tree) == source + project = Project(str(tmp_path)) + try: + module = project.root.create_file("consumer.py") + module.write(source) + finder = similarfinder.SimilarFinder(project.get_pymodule(module)) + matches = list(finder.get_matches("(\nself._record_execution_metrics\n)")) + assert [match.get_region() for match in matches] == [expected] + finally: + project.close() + + +@pytest.mark.parametrize("prefix", ["", "\u03c0"]) +def test_fstring_fields_follow_expression_coordinates(prefix): + source = dedent( + 'result = f"${{frozen}}({{{literal}}})" if frozen else literal\n' + ).replace('f"', f'f"{prefix}', 1) + module = patchedast.get_patched_ast(source, sorted_children=True) + fields = [node for node in ast.walk(module) if isinstance(node, ast.FormattedValue)] + start = source.index("{literal}") + assert len(fields) == 1 + assert fields[0].region == (start, start + len("{literal}")) + assert patchedast.write_ast(module) == source + + +class CapturedFileSystem(FileSystemCommands): + """A read-only filesystem captured from real test inputs.""" + + def __init__(self, roots): + self.files = {} + self.directories = {} + self.states = {} + for root in roots: + for directory, _folders, files in os.walk(root): + self.directories[directory] = tuple(os.listdir(directory)) + self.states[directory] = os.stat(directory) + for name in files: + path = os.path.join(directory, name) + self.states[path] = os.stat(path) + with open(path, "rb") as stream: + self.files[path] = stream.read() + + @staticmethod + def realpath(path): + return os.path.abspath(os.path.expanduser(path)) + + def exists(self, path): + return path in self.files or path in self.directories + + def isfile(self, path): + return path in self.files + + def isdir(self, path): + return path in self.directories + + def listdir(self, path): + return self.directories[path] + + def read(self, path): + return self.files[path] + + def stat(self, path): + return self.states[path] + + +@pytest.mark.parametrize("captured", [False, True]) +@pytest.mark.parametrize( + "destination, expected", + [("", "source.py"), ("existing", "existing/source.py"), ("late", "late")], +) +def test_move_destination_uses_project_filesystem( + tmp_path, captured, destination, expected +): + root = tmp_path / "project" + (root / "existing").mkdir(parents=True) + (root / "source.py").write_text("value = 1\n", encoding="utf-8") + commands = CapturedFileSystem((str(root),)) if captured else FileSystemCommands() + if captured: + shutil.rmtree(root) + (root / "late").mkdir(parents=True) + project = Project( + str(root), + fscommands=commands, + ropefolder=None, + save_objectdb=False, + save_history=False, + ) + try: + source = project.get_resource("source.py") + assert change.MoveResource(source, destination).new_resource.path == expected + assert ( + change.MoveResource(source, destination, exact=True).new_resource.path + == destination + ) + finally: + project.close() + + +def test_local_and_external_snapshot_resources_ignore_changed_disk(tmp_path): + root = tmp_path / "consumer" + local = root / "src" + external = tmp_path / "dependencies" + package = external / "provider" + local.mkdir(parents=True) + package.mkdir(parents=True) + (local / "consumer.py").write_text( + "from provider import Original\nborrowed = Original\n", + encoding="utf-8", + ) + (package / "__init__.py").write_text( + "class Original:\n marker = 1\n", + encoding="utf-8", + ) + captured = CapturedFileSystem((str(root), str(external))) + singleton = get_no_project() + shutil.rmtree(root) + shutil.rmtree(external) + package.mkdir(parents=True) + (package / "__init__.py").write_text( + "class Changed:\n marker = 2\n", + encoding="utf-8", + ) + (external / "late.py").write_text("marker = 3\n", encoding="utf-8") + outside = NoProject(fscommands=captured) + project = Project( + str(root), + fscommands=captured, + external_project=outside, + ropefolder=None, + source_folders=["src"], + python_path=[str(external)], + save_objectdb=False, + save_history=False, + ) + try: + module = project.get_module("consumer") + name = module.get_attribute("borrowed") + assert name.get_object().get_name() == "Original" + holder = name.get_object().get_module() + assert holder.get_resource().project is outside + assert project.find_module("late") is None + assert project.get_resource("src/consumer.py").read().startswith( + "from provider import Original" + ) + assert project.get_folder("src").get_children() == [ + project.get_file("src/consumer.py") + ] + assert get_no_project() is singleton + assert not root.exists() + finally: + project.close() + + +def test_snapshot_directory_acquisition_failure_is_not_an_empty_result(tmp_path): + root = tmp_path / "consumer" + root.mkdir() + captured = CapturedFileSystem((str(root),)) + project = Project( + str(root), + fscommands=captured, + external_project=NoProject(fscommands=captured), + ropefolder=None, + save_objectdb=False, + save_history=False, + ) + try: + del captured.directories[str(root)] + with pytest.raises(KeyError): + project.get_folder("").get_children() + finally: + project.close() + + +def test_native_relative_namespace_and_stub_resources_use_captured_inputs(tmp_path): + root = tmp_path / "consumer" + local = root / "src" + package = local / "application" + external = tmp_path / "dependencies" + namespace = external / "namespace" + package.mkdir(parents=True) + namespace.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "owner.py").write_text("class Local:\n pass\n", encoding="utf-8") + (package / "consumer.py").write_text( + "from .owner import Local\n" + "from namespace.owner import External\n" + "local = Local\nexternal = External\n", + encoding="utf-8", + ) + (namespace / "owner.py").write_text( + "class External:\n pass\n", encoding="utf-8" + ) + (local / "schema.pyi").write_text( + "class Schema:\n value: str\n", encoding="utf-8" + ) + captured = CapturedFileSystem((str(root), str(external))) + shutil.rmtree(root) + shutil.rmtree(external) + outside = NoProject(fscommands=captured) + project = Project( + str(root), + fscommands=captured, + external_project=outside, + ropefolder=None, + source_folders=["src"], + python_path=[str(external)], + save_objectdb=False, + save_history=False, + ) + try: + consumer = project.get_module("application.consumer") + relative = project.find_relative_module( + "owner", project.get_folder("src/application"), 1 + ) + assert relative == project.get_resource("src/application/owner.py") + local_owner = consumer.get_attribute("local").get_object() + assert local_owner.get_name() == "Local" + assert local_owner.get_module().get_resource() == relative + external_owner = consumer.get_attribute("external").get_object() + assert external_owner.get_name() == "External" + assert external_owner.get_module().get_resource().project is outside + assert project.find_module("namespace").is_folder() + stub = project.get_resource("src/schema.pyi") + assert project.get_pymodule(stub).get_attribute("Schema").get_object().get_name() == "Schema" + # This port preserves Rope's existing module election: .pyi is an + # explicitly supplied resource, not an extra module-name fallback. + assert project.find_module("schema") is None + assert not root.exists() + assert not external.exists() + finally: + project.close() diff --git a/ropetest/source_ancestry_fixtures.py b/ropetest/source_ancestry_fixtures.py new file mode 100644 index 000000000..4cae589c4 --- /dev/null +++ b/ropetest/source_ancestry_fixtures.py @@ -0,0 +1,38 @@ +"""Maintained runtime examples for comparison with native source inference.""" + +import json + + +class First: + pass + + +class Sibling: + pass + + +def identity(value): + return value + + +selected = First +if False: + selected = Sibling +else: + selected = identity(selected) + + +class Derived(selected): + pass + + +if __name__ == "__main__": + print( + json.dumps( + { + "selected": selected.__qualname__, + "ancestry": [base.__qualname__ for base in Derived.__mro__], + }, + indent=2, + ) + ) diff --git a/ropetest/source_ancestry_probe.py b/ropetest/source_ancestry_probe.py new file mode 100644 index 000000000..ced841ac7 --- /dev/null +++ b/ropetest/source_ancestry_probe.py @@ -0,0 +1,324 @@ +"""Inspect native ancestry against explicit source roots without importing targets.""" + +import argparse +import json +from pathlib import Path + +from rope.base import ast, evaluate, pynames, pynamesdef, pyobjects, pyscopes +from rope.base.project import Project + + +def describe(value): + """Describe existing native identities without resolving another binding. + + Object and type ids are process-local witnesses. Type module and resource + metadata is included only when the native object exposes that origin; + unresolved kinds do not acquire an assumed name or declaring module. + Getter failures propagate instead of substituting a different object. + """ + if value is None: + return None + result = { + "native_type": type(value).__qualname__, + "native_id": hex(id(value)), + } + if isinstance(value, (pyobjects.AbstractClass, pyobjects.PyFunction)): + result["name"] = value.get_name() + module = value.get_module() + if module is not None: + result["module"] = module.get_name() + if isinstance(value, pyobjects.PyObject): + value_type = value.get_type() + result["object_type"] = type(value_type).__qualname__ + result["object_type_id"] = hex(id(value_type)) + if isinstance(value_type, pyobjects.AbstractClass): + result["object_type_name"] = value_type.get_name() + type_module = value_type.get_module() + if type_module is not None: + result["object_type_module"] = type_module.get_name() + resource = type_module.get_resource() + if resource is not None: + result["object_type_resource"] = resource.real_path + if isinstance(value, pyobjects.AbstractFunction): + result["name"] = value.get_name() + return result + + +def describe_base(owner, base): + result = describe(base) + if isinstance(base, pyobjects.UnresolvedBase): + result["expression"] = ast.unparse(base.node) + result["inferred"] = describe(base.pyobject) + base_binding = evaluate.eval_node(owner.parent.get_scope(), base.node) + if base_binding is not None: + base_object = base_binding.get_object() + result["base_binding"] = describe(base_object) + result["has_mro_entries"] = ( + "__mro_entries__" in base_object.get_attributes() + ) + if isinstance(base.node, ast.Subscript): + binding = evaluate.eval_node(owner.parent.get_scope(), base.node.value) + if binding is not None: + receiver = binding.get_object() + result["subscription_receiver"] = describe(receiver) + hook = receiver.get_attributes().get("__class_getitem__") + if hook is not None: + function = hook.get_object() + result["subscription_hook"] = describe(function) + module = function.get_module() + if module is not None: + result["hook_module"] = module.get_name() + if isinstance(function, pyobjects.PyFunction): + scope = function.get_scope() + result["hook_parameters"] = { + name: describe(parameter.get_object()) + for name, parameter in function.get_parameters().items() + } + result.update(describe_parameter_flow(function)) + result["hook_bindings"] = { + name: { + "assignments": [ + ast.unparse(assignment.ast_node) + for assignment in binding.assignments + ], + "inferred": describe(binding.get_object()), + } + for name, binding in scope.get_names().items() + if isinstance(binding, pynames.AssignedName) + } + result["hook_returns"] = [] + for expression in scope._get_returned_asts(): + returned = evaluate.eval_node(scope, expression) + result["hook_returns"].append( + { + "expression": ast.unparse(expression), + "inferred": describe( + returned.get_object() + if returned is not None + else None + ), + } + ) + result["assigned_call_frames"] = [] + for binding in scope.get_names().values(): + if not isinstance(binding, pynames.AssignedName): + continue + for assignment in binding.assignments: + expression = assignment.ast_node + if not isinstance(expression, ast.Call): + continue + called = evaluate.eval_node(scope, expression.func) + if called is None: + continue + callable_object = called.get_object() + if isinstance(callable_object, pyobjects.PyFunction): + result["assigned_call_frames"].append( + describe_frame(callable_object) + ) + return result + + +def describe_frame(function, preassignment_names=()): + """Report a native function frame and its elected module resource, or None.""" + module = function.get_module() + scope = function.get_scope() + resource = module.get_resource() + result = { + "name": function.get_name(), + "module": module.get_name(), + "resource": resource.real_path if resource is not None else None, + "parameters": { + name: describe(parameter.get_object()) + for name, parameter in function.get_parameters().items() + }, + "locals": { + name: { + "assignments": [ + ast.unparse(assignment.ast_node) + for assignment in binding.assignments + ], + "inferred": describe(binding.get_object()), + "assignment_values": [ + describe_expression(scope, assignment.ast_node) + for assignment in binding.assignments + ], + } + for name, binding in scope.get_names().items() + if isinstance(binding, pynames.AssignedName) + }, + } + result["preassignment_witnesses"] = { + name: describe_preassignment(scope, scope.get_name(name)) + for name in preassignment_names + } + return result + + +def describe_parameter_flow(function): + module = function.get_module() + scope = function.get_scope() + parameters = function.get_parameters() + result = [] + reads = [] + for context in module.get_source_binding_index().node_contexts.values(): + node = context.source_node + if not isinstance(node, (ast.Assign, ast.AnnAssign)): + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id in parameters + and context.structural_scope.pyobject is function + ): + reads.append(node) + continue + if context.structural_scope.pyobject is not function: + continue + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + if not isinstance(target, ast.Name) or target.id not in parameters: + continue + binder = scope.get_name(target.id) + result.append( + { + "parameter": target.id, + "binder_type": type(binder).__qualname__, + "is_formal_binder": binder is parameters[target.id], + "formal_value": describe(parameters[target.id].get_object()), + "scope_binding_value": describe(binder.get_object()), + "assignment": ( + describe_expression(scope, node.value) + if node.value is not None + else None + ), + } + ) + written_names = {write["parameter"] for write in result} + return { + "parameter_writes": result, + "parameter_reads": [ + { + "line": node.lineno, + "column": node.col_offset, + **describe_expression(scope, node), + } + for node in reads + if node.id in written_names + ], + } + + +def describe_preassignment(scope, binding): + if not isinstance(binding, pynames.AssignedName) or len(binding.assignments) < 2: + raise TypeError("A preassignment witness requires multiple native assignments") + original = describe(binding.get_object()) + binding.invalidate() + invalidated = describe(binding.get_object()) + prefix = pynamesdef.AssignedName(module=binding.module) + prefix.assignments = binding.assignments[:-1] + name = next(name for name, value in scope.get_names().items() if value is binding) + temporary = pyscopes.TemporaryScope(scope.pycore, scope, {name: prefix}) + return { + "original": original, + "after_local_invalidation": invalidated, + "previous_assignments": describe(prefix.get_object()), + "last_expression_with_previous_binding": describe_expression( + temporary, binding.assignments[-1].ast_node + ), + } + + +def describe_expression(scope, expression): + binding = evaluate.eval_node(scope, expression) + return { + "expression": ast.unparse(expression), + "inferred": describe(binding.get_object() if binding is not None else None), + } + + +def describe_binding(module, binding): + """Report a native binding and its elected module resource, or None.""" + result = {"value": describe(binding.get_object())} + resource = module.get_resource() + result["resource"] = resource.real_path if resource is not None else None + if isinstance(binding, pynames.AssignedName): + scope = module.get_scope() + result["calls"] = [] + for assignment in binding.assignments: + expression = assignment.ast_node + if not isinstance(expression, ast.Call): + continue + call = describe_expression(scope, expression) + call["callable"] = describe_expression(scope, expression.func) + if isinstance(expression.func, ast.Attribute): + call["receiver"] = describe_expression(scope, expression.func.value) + call["arguments"] = [ + describe_expression(scope, argument) for argument in expression.args + ] + result["calls"].append(call) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("request", type=Path) + options = parser.parse_args() + with options.request.open(encoding="utf-8") as source: + request = json.load(source) + project = Project( + request["root"], + ropefolder=None, + python_path=request["python_path"], + save_history=False, + save_objectdb=False, + ) + try: + for target in request.get("bindings", []): + module = project.get_module(target["module"]) + holder = module + for name in target["attributes"][:-1]: + holder = holder.get_attribute(name).get_object() + binding = holder.get_attribute(target["attributes"][-1]) + print( + json.dumps( + {"target": target, **describe_binding(module, binding)}, + ensure_ascii=False, + indent=2, + ), + flush=True, + ) + for target in request["targets"]: + module = project.get_module(target["module"]) + value = module + for name in target["attributes"]: + value = value.get_attribute(name).get_object() + if not isinstance(value, pyobjects.AbstractClass): + raise TypeError("The requested binding is not a native class") + resource = module.get_resource() + result = { + "target": target, + "class": describe(value), + "resource": resource.real_path if resource is not None else None, + "bases": [describe_base(value, base) for base in value.get_bases()], + "mro": [describe_base(value, base) for base in value.get_mro()], + } + print(json.dumps(result, ensure_ascii=False, indent=2), flush=True) + for target in request.get("frames", []): + function = project.get_module(target["module"]) + for name in target["attributes"]: + function = function.get_attribute(name).get_object() + if not isinstance(function, pyobjects.PyFunction): + raise TypeError("The requested frame is not a native source function") + print( + json.dumps( + describe_frame(function, target.get("preassignment_names", [])), + ensure_ascii=False, + indent=2, + ), + flush=True, + ) + finally: + project.close() + + +if __name__ == "__main__": + main() diff --git a/ropetest/source_assignment_binding_test.py b/ropetest/source_assignment_binding_test.py new file mode 100644 index 000000000..5088fd8fe --- /dev/null +++ b/ropetest/source_assignment_binding_test.py @@ -0,0 +1,49 @@ +"""Source binding identity for assignment targets and call keywords.""" + +from textwrap import dedent + +from rope.base import codeanalyze, evaluate +from rope.base.project import Project + + +def test_unpack_targets_resolve_to_later_references_and_preserve_keywords(tmp_path): + project = Project(str(tmp_path), save_objectdb=False, save_history=False) + try: + source = dedent('''\ + def receive(option=None): + return option + + def process(value): + título = "á" + first, second = value, value + (left, (middle, + last)) = (value, (value, value)) + receive(option=last) + return first, second, left, middle, last, título + ''') + resource = project.root.create_file('subject.py') + resource.write(source) + module = project.get_pymodule(resource) + for name, target in ( + ('second', 'second = value'), + ('last', 'last)) ='), + ('título', 'título ='), + ): + declared = evaluate.eval_location(module, source.index(target)) + used = evaluate.eval_location(module, source.rindex(name)) + assert declared is not None + assert declared is used + parameter = evaluate.eval_location(module, source.index('option=None')) + keyword = evaluate.eval_location(module, source.index('option=last')) + assert parameter is not None + assert keyword is parameter + finally: + project.close() + + +def test_name_offsets_exclude_literals_and_comments_and_include_fstrings(): + source = 'árvore = 1 # árvore\ntext = "árvore"\nprint(fr"{árvore}")\n' + assert codeanalyze.name_offsets(source, 'árvore') == ( + source.index('árvore'), + source.rindex('árvore'), + ) diff --git a/ropetest/source_model_master_test.py b/ropetest/source_model_master_test.py new file mode 100644 index 000000000..5f4a03602 --- /dev/null +++ b/ropetest/source_model_master_test.py @@ -0,0 +1,1574 @@ +"""Source identities on the unmodified ordinary-function argument protocol.""" + +import sys +import typing +from textwrap import dedent, indent + +import pytest + +from rope.base import ast, evaluate, exceptions, pyobjects +from rope.base.project import Project +from rope.base.sourcebindings import SourceBindingIndex +from rope.contrib import codeassist, finderrors +from rope.refactor import introduce_parameter, rename + + +@pytest.fixture +def model_project(tmp_path): + project = Project( + str(tmp_path), save_objectdb=False, save_history=False, automatic_soa=False + ) + yield project + project.close() + + +def module(project, source): + resource = project.root.create_file("subject.py") + resource.write(source) + return resource + + +def test_definition_queries_complete_the_same_index_on_body_demand(model_project): + source = dedent("""\ + class Base: + def unused(self): + return [value for value in range(3)] + class Derived(Base): + pass + def outer(value): + def inner(): + return value + return inner + """) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + derived = pymodule["Derived"].get_object() + assert [owner.get_name() for owner in derived.get_mro()] == ["Derived", "Base"] + index = pymodule._get_source_binding_index() + unused = pymodule.get_ast().body[0].body[0] + assert index.node_contexts.get(id(unused.body[0])) is None + + outer = pymodule["outer"].get_object() + closure = evaluate.eval_location(pymodule, source.index("return value") + 7) + assert closure is outer.get_parameters()["value"] + assert index.node_contexts.get(id(unused.body[0])) is None + + assert pymodule.get_source_binding_index() is index + assert id(unused.body[0]) in index.node_contexts + eager = SourceBindingIndex(pymodule) + assert set(index.node_contexts) == set(eager.node_contexts) + for key, context in eager.node_contexts.items(): + actual = index.node_contexts[key] + assert actual.role == context.role + assert actual.environment.get_kind() == context.environment.get_kind() + assert actual.structural_scope is context.structural_scope + + +def test_nested_function_binding_children_materialize_their_body(model_project): + source = dedent("""\ + def outer(): + class Nested: + callback = lambda item: [value for value in item] + return Nested + """) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + outer = pymodule["outer"].get_object() + nested = outer.get_scope().get_name("Nested").get_object() + assert nested.get_scope().get_binding_scopes() == () + (callback,) = outer.get_scope().get_binding_scopes() + assert callback.get_kind() == "Lambda" + (comprehension,) = callback.get_binding_scopes() + assert comprehension.get_kind() == "Comprehension" + assert evaluate.eval_location( + pymodule, source.index("[value") + 1 + ) is comprehension.get_name("value") + + +@pytest.mark.parametrize("declaration", ["def inner():", "async def inner():", "class Inner:"]) +@pytest.mark.parametrize("query", ["offset", "node"]) +def test_first_nested_decorator_preserves_lexical_binding( + model_project, declaration, query +): + source = dedent(f"""\ + def outer(): + @(lambda decorated: decorated) + {declaration} + pass + """) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + decorator = pymodule.get_ast().body[0].body[0].decorator_list[0] + if query == "node": + index = pymodule._get_source_binding_index() + context = index.context_for_node(decorator.body) + assert context is not None + binding = context.environment.get_name("decorated") + else: + binding = evaluate.eval_location(pymodule, source.index("decorated:") + 1) + assert binding is not None + assert evaluate.eval_location( + pymodule, source.index(": decorated") + 3 + ) is binding + assert evaluate.eval_location( + pymodule, source.index("decorated:") + 1 + ) is binding + assert pymodule.get_source_binding_index().context_for_node( + decorator.body + ).environment.get_name("decorated") is binding + + +def test_public_scope_enumeration_includes_nested_future_annotations(model_project): + source = dedent("""\ + from __future__ import annotations + def outer(): + def inner(value: int): + pass + """) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + scopes = pymodule.get_scope().get_binding_scopes() + assert [(scope.get_kind(), scope.node.lineno) for scope in scopes] == [ + ("Annotation", 3) + ] + evaluate.eval_location(pymodule, source.index("value:") + 1) + assert pymodule.get_scope().get_binding_scopes() == scopes + pymodule.get_source_binding_index() + assert pymodule.get_scope().get_binding_scopes() == scopes + + +def prefix(mode): + if mode == "eager" and sys.version_info >= (3, 14): + pytest.skip("eager function annotations require Python before 3.14") + if mode == "lazy" and sys.version_info < (3, 14): + pytest.skip("lazy function annotations require Python 3.14") + return "from __future__ import annotations\n" if mode == "future" else "" + + +def observe(source): + namespace = {"__name__": "fixture"} + exec(compile(source, "fixture.py", "exec"), namespace) + return namespace["drive"](), tuple( + typing.get_type_hints(namespace["subject"]).values() + ) + + +@pytest.mark.parametrize("mode", ["eager", "future", "lazy"]) +@pytest.mark.parametrize("owner", ["module", "class", "nested"]) +@pytest.mark.parametrize("operation", ["outer", "formal"]) +def test_existing_rename_api_keeps_runtime_annotation_contract( + model_project, mode, owner, operation +): + source = prefix(mode) + "target = int\n" + definition = "def func(target: target) -> target:\n return target\n" + if owner == "module": + source += definition + "subject = func\n" + outer = source.index("target = int") + elif owner == "class": + source += "class Owner:\n" + indent("target = bytes\n" + definition, " ") + source += "subject = Owner.func\n" + outer = source.index("target = bytes") + else: + source += "def owner():\n" + indent("target = bytes\n" + definition, " ") + source += " return func\nsubject = owner()\n" + outer = source.index("target = bytes") + source += "def drive():\n return subject(str)\n" + before = observe(source) + resource = module(model_project, source) + offset = ( + outer + if operation == "outer" + else source.index("target", source.index("def func(")) + ) + changes = rename.Rename(model_project, resource, offset).get_changes("renamed") + model_project.do(changes) + assert observe(resource.read()) == before + + +@pytest.mark.parametrize( + "signature,call", + [("target", "str"), ("target, /", "str"), ("*, target", "target=str")], +) +def test_lambda_formal_rename_preserves_keyword_driver(model_project, signature, call): + source = f"target = int\ncallback = lambda {signature}: target\nresult = callback({call})\n" + namespace = {} + exec(source, namespace) + before = namespace["result"] + resource = module(model_project, source) + offset = source.index("target", source.index("lambda ")) + changes = rename.Rename(model_project, resource, offset).get_changes("renamed") + model_project.do(changes) + namespace = {} + exec(resource.read(), namespace) + assert namespace["result"] is before is str + assert "target = int" in resource.read() + + +@pytest.mark.parametrize( + "expression", + [ + "[target for target in [str]]", + "{target for target in [str]}", + "{target: target for target in [str]}", + "tuple(target for target in [str])", + ], +) +def test_outer_rename_does_not_capture_comprehension_local(model_project, expression): + source = f"target = int\nvalue = {expression}\nresult = (target, value)\n" + before = {} + exec(source, before) + resource = module(model_project, source) + changes = rename.Rename(model_project, resource, 0).get_changes("renamed") + model_project.do(changes) + after = {} + exec(resource.read(), after) + assert after["result"] == before["result"] + assert "for target in" in resource.read() + + +@pytest.mark.parametrize( + "signature,call", [("target, /", "str"), ("*, target", "target=str")] +) +def test_ordinary_formal_kinds_use_parameter_protocol_bindings( + model_project, signature, call +): + # The source binding index keeps these formals distinct from same-named + # outer variables (#900) while the ordinary parameter protocol owns their + # identity and refactoring (#898): both hold after the reconciliation. + source = f"target = int\ndef func({signature}):\n return target\nresult = (target, func({call}))\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + function = pymodule["func"].get_object() + formal_offset = source.index("target", source.index("def func(")) + binder = evaluate.eval_location(pymodule, formal_offset) + assert binder is function.get_parameters()["target"] + assert binder is not pymodule["target"] + assert ( + evaluate.eval_location( + pymodule, source.index("target", source.index("return ")) + ) + is binder + ) + assert function.get_param_names() == ["target"] + assert codeassist.get_definition_location( + model_project, source, formal_offset, resource + ) == (resource, 2) + assert finderrors.find_errors(model_project, resource) == [] + changes = rename.Rename(model_project, resource, 0).get_changes("outer") + model_project.do(changes) + namespace = {} + exec(resource.read(), namespace) + assert namespace["result"] == (int, str) + assert "return target" in resource.read() + + +@pytest.mark.parametrize( + "signature,call", [("target, /", "str"), ("*, target", "target=str")] +) +@pytest.mark.parametrize("selected", ["formal", "body"]) +def test_ordinary_formal_kinds_rename_through_parameter_protocol( + model_project, signature, call, selected +): + source = f"target = int\ndef func({signature}):\n return target\nresult = (target, func({call}))\n" + resource = module(model_project, source) + before = {} + exec(source, before) + offset = source.index( + "target", source.index("def func(") if selected == "formal" else source.index("return ") + ) + model_project.do(rename.Rename(model_project, resource, offset).get_changes("renamed")) + after = {} + exec(resource.read(), after) + assert after["result"] == before["result"] == (int, str) + assert "return renamed" in resource.read() + assert "target = int" in resource.read() + + +@pytest.mark.parametrize( + "expression,expected", + [ + ("(lambda target=str: target)()", "str"), + ("(lambda *, target=str: target)()", "str"), + ("(lambda target=int, /, **kw: target)(target=str)", "int"), + ("(lambda target: target)(str)", "str"), + ("(lambda *, target: target)(target=str)", "str"), + ("(lambda target=int: target)(*[str])", None), + ("(lambda target=int: target)(**{'target': str})", None), + ("(lambda target: target)(str, target=int)", None), + ("(lambda target: target)(str, **{'target': int})", None), + ("(lambda target: target)(*[str], target=int)", None), + ("(lambda target=int: target)(missing)", None), + ], +) +def test_lambda_private_matching_does_not_use_ordinary_function_api( + model_project, expression, expected +): + source = "result = " + expression + "\n" + pymodule = model_project.get_pymodule(module(model_project, source)) + result = pymodule["result"].get_object() + if expected is None: + assert result is pyobjects.get_unknown() + else: + from rope.base import builtins + + assert result is builtins.builtins[expected].get_object() + node = next(n for n in ast.walk(pymodule.get_ast()) if isinstance(n, ast.Lambda)) + function = pymodule.get_source_binding_index().callable_for_node(node) + canonical = function.get_parameters()["target"] + assert canonical.get_object() is pyobjects.get_unknown() + model_project.pycore.module_cache.forget_all_data() + assert function.get_parameters()["target"] is canonical + + +@pytest.mark.parametrize( + "expression", + [ + "Owner().callback(item=Other()).field", + "Owner.callback(Other(), item=Owner()).field", + "Owner().callback(Other()).field", + ], +) +def test_lambda_descriptor_field_rename_keeps_receiver(model_project, expression): + source = ( + "class Other:\n field = 2\nclass Owner:\n field = 1\n callback = lambda self, item: self\nresult = " + + expression + + "\n" + ) + before = {} + exec(source, before) + resource = module(model_project, source) + changes = rename.Rename( + model_project, resource, source.index("field = 1") + ).get_changes("renamed") + model_project.do(changes) + after = {} + exec(resource.read(), after) + assert after["result"] == before["result"] + assert "field = 2" in resource.read() + + +@pytest.mark.parametrize( + "source", + [ + "target = str\ndef func(target=target):\n return target\n", + "target = str\ndef func(target: target):\n return target\n", + ], +) +def test_first_function_names_do_not_visit_header_into_local_scope( + model_project, source +): + pymodule = model_project.get_pymodule(module(model_project, source)) + scope = pymodule["func"].get_object().get_scope() + formal = scope.pyobject.get_parameters()["target"] + assert scope.get_names()["target"] is formal + assert scope.lookup("target") is formal + assert ( + evaluate.eval_location( + pymodule, source.index("target", source.index("return ")) + ) + is formal + ) + assert ( + evaluate.eval_location( + pymodule, + ( + source.index("target", source.index("target=") + 7) + if "target=" in source + else source.index("target", source.index(": target")) + ), + ) + is pymodule["target"] + ) + + +def test_introduce_parameter_keeps_structural_body_and_refuses_lambda(model_project): + source = ( + "value = 1\ndef func():\n callback = lambda item: value\n return value\n" + ) + resource = module(model_project, source) + with pytest.raises(exceptions.RefactoringError, match="inside functions"): + introduce_parameter.IntroduceParameter( + model_project, resource, source.index("item", source.index("lambda")) + ) + capture = introduce_parameter.IntroduceParameter( + model_project, resource, source.index("value", source.index("lambda")) + ) + model_project.do(capture.get_changes("captured")) + assert "def func(captured=value)" in resource.read() + assert "lambda item: captured" in resource.read() + resource.write(source) + operation = introduce_parameter.IntroduceParameter( + model_project, resource, source.index("value", source.index("return")) + ) + model_project.do(operation.get_changes("provided")) + assert "def func(provided=value)" in resource.read() + assert "lambda item: provided" in resource.read() + + +@pytest.mark.parametrize( + "signature", ["target", "target, /", "*, target", "*target", "**target"] +) +def test_source_lambda_index_keeps_all_formal_identities_across_cache_reset( + model_project, signature +): + source = f"target = int\ncallback = lambda {signature}: target\n" + pymodule = model_project.get_pymodule(module(model_project, source)) + index = pymodule.get_source_binding_index() + node = next(n for n in ast.walk(pymodule.get_ast()) if isinstance(n, ast.Lambda)) + callable_ = index.callable_for_node(node) + binder = callable_.get_parameters()["target"] + offset = source.index("target", source.index("lambda")) + assert index.binding_for_token(offset) is binder + assert evaluate.eval_location(pymodule, offset) is binder + body_offset = source.rindex("target") + assert evaluate.eval_location(pymodule, body_offset) is binder + assert binder is not pymodule["target"] + assert index.context_at(body_offset).structural_scope is pymodule.get_scope() + model_project.pycore.module_cache.forget_all_data() + assert pymodule.get_source_binding_index() is index + assert callable_.get_parameters()["target"] is binder + assert evaluate.eval_location(pymodule, body_offset) is binder + + +@pytest.mark.parametrize( + "source,offset_text", + [ + ("target = int\ndef func(target: target):\n return target\n", ": target"), + ("target = int\ndef func(target=target):\n return target\n", "=target"), + ("target = int\ncallback = lambda target=target: target\n", "=target"), + ], +) +def test_change_occurrences_uses_structure_or_canonical_expression_scope( + model_project, source, offset_text +): + resource = module(model_project, source) + offset = source.index(offset_text) + 1 + if source[offset] == " ": + offset += 1 + operation = rename.ChangeOccurrences(model_project, resource, offset) + model_project.do(operation.get_changes("changed")) + if "def func" in source: + assert "target = int" in resource.read() + assert ( + "target: changed" in resource.read() or "target=changed" in resource.read() + ) + else: + assert "changed = int" in resource.read() + assert "return target" in resource.read() or ": target" in resource.read() + + +@pytest.mark.parametrize("kind", ["global", "nonlocal"]) +def test_header_named_expression_uses_declared_owner_before_first_lookup( + model_project, kind +): + if kind == "global": + source = "target = int\ndef owner():\n global target\n def inner(value=(target := str)):\n return target\n return inner\nresult = owner()()\n" + owner_offset = source.index("target = int") + else: + source = "def enclosing():\n target = int\n def owner():\n nonlocal target\n def inner(value=(target := str)):\n return target\n return inner\n return owner()\nresult = enclosing()()\n" + owner_offset = source.index("target = int") + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + binder = evaluate.eval_location(pymodule, owner_offset) + walrus = source.index("target :=") + assert evaluate.eval_location(pymodule, walrus) is binder + model_project.do( + rename.Rename(model_project, resource, owner_offset).get_changes("renamed") + ) + namespace = {} + exec(resource.read(), namespace) + assert namespace["result"] is str + assert "target" not in resource.read() + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="type aliases require Python 3.12" +) +def test_alias_own_parameter_is_distinct_and_rename_keeps_runtime_value(model_project): + source = "T = int\ntype Alias[T] = tuple[T, T]\nresult = Alias.__value__\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + own_offset = source.index("[T]") + 1 + binder = evaluate.eval_location(pymodule, own_offset) + assert binder is not pymodule["T"] + assert evaluate.eval_location(pymodule, source.index("[T, T]") + 1) is binder + model_project.do( + rename.Rename(model_project, resource, own_offset).get_changes("V") + ) + namespace = {} + exec(resource.read(), namespace) + alias = namespace["Alias"] + assert namespace["T"] is int + assert alias.__value__.__args__ == ( + alias.__type_params__[0], + alias.__type_params__[0], + ) + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="type parameters require Python 3.12" +) +def test_generic_class_collision_uses_opaque_source_and_refuses_atomically( + model_project, +): + source = "T = int\nclass Owner[T]:\n T = str\n def method(self, value: T):\n return value\n" + resource = module(model_project, source) + for offset in (source.index("[T]") + 1, source.index("T = str")): + with pytest.raises(exceptions.RefactoringError, match="unresolved"): + rename.Rename(model_project, resource, offset).get_changes("V") + assert resource.read() == source + + +@pytest.mark.parametrize("signature", ["target, /", "*, target"]) +def test_introduce_parameter_refuses_unsupported_ordinary_signature_atomically( + model_project, signature +): + source = f"value = int\ndef func({signature}):\n return value\n" + resource = module(model_project, source) + operation = introduce_parameter.IntroduceParameter( + model_project, resource, source.index("value", source.index("return")) + ) + # Both #898 and #900 refuse these signatures; the merged stack raises + # the functionutils guard shared by every signature-rewriting operation. + with pytest.raises( + exceptions.RefactoringError, + match="does not support positional-only or keyword-only parameters", + ): + operation.get_changes("provided") + assert resource.read() == source + + +@pytest.mark.parametrize( + "source", + [ + "target = int\ncallback = lambda target: target\n", + "target = int\nresult = [target for target in [str]]\n", + ], +) +def test_completion_follows_canonical_expression_environment(model_project, source): + resource = module(model_project, source) + offset = ( + source.index("target", source.index(": target") + 2) + 3 + if "lambda" in source + else source.index("target", source.index("result")) + 3 + ) + proposals = codeassist.code_assist( + model_project, source, offset, resource=resource, later_locals=False + ) + assert "target" in {proposal.name for proposal in proposals} + assert finderrors.find_errors(model_project, resource) == [] + + +def test_old_offset_scope_remains_structural_while_index_resolves_comp_identity( + model_project, +): + source = "target = int\nresult = [target for target in [str]]\n" + pymodule = model_project.get_pymodule(module(model_project, source)) + scope = pymodule.get_scope() + offset = source.index("target", source.index("result")) + assert scope.get_inner_scope_for_offset(offset) is scope.get_scopes()[0] + assert scope.get_structural_scope_for_offset(offset) is scope.get_scopes()[0] + canonical = pymodule.get_source_binding_index().context_at(offset).environment + assert canonical is not scope.get_scopes()[0] + assert evaluate.eval_location(pymodule, offset) is canonical.lookup("target") + assert evaluate.eval_location(pymodule, offset) is not pymodule["target"] + + +@pytest.mark.parametrize("class_first", [False, True]) +@pytest.mark.parametrize("class_return", ["self", "item", "static"]) +@pytest.mark.parametrize("access", ["direct", "saved", "class", "class-saved"]) +@pytest.mark.parametrize("renamed_owner", ["Owner", "Other"]) +def test_lambda_storage_view_selects_instance_and_class_values_independently( + model_project, class_first, class_return, access, renamed_owner +): + initializer = ( + " def __init__(self):\n" + " self.field = 1\n" + " self.callback = lambda item: item\n" + ) + class_value = ( + " callback = staticmethod(lambda item: item)\n" + if class_return == "static" + else f" callback = lambda self, item: {class_return}\n" + ) + source = "class Owner:\n" + ( + class_value + initializer if class_first else initializer + class_value + ) + source += "class Other:\n def __init__(self):\n self.field = 2\n" + if access == "direct": + source += "result = Owner().callback(Other())\n" + elif access == "saved": + source += "saved = Owner().callback\nresult = saved(Other())\n" + elif access == "class": + source += ( + "result = Owner.callback(Other())\n" + if class_return == "static" + else "result = Owner.callback(Owner(), Other())\n" + ) + else: + source += "saved = Owner.callback\n" + source += ( + "result = saved(Other())\n" + if class_return == "static" + else "result = saved(Owner(), Other())\n" + ) + source += "observed = result.field\n" + before = {} + exec(compile(source, "storage_fixture.py", "exec"), before) + expected_owner = ( + "Owner" + if access in {"class", "class-saved"} and class_return == "self" + else "Other" + ) + assert before["observed"] == (1 if expected_owner == "Owner" else 2) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + # Reading either namespace first must not change the other value view. + first = "Owner.callback" if class_first else "Owner().callback" + second = "Owner().callback" if class_first else "Owner.callback" + first_view = evaluate.eval_str(pymodule.get_scope(), first) + first_function = first_view.get_object() + first_parameters = first_function.get_parameters() + second_view = evaluate.eval_str(pymodule.get_scope(), second) + second_function = second_view.get_object() + assert first_function.source_function is not second_function.source_function + assert first_view.get_object() is first_function + assert first_function.get_parameters() == first_parameters + assert ( + evaluate.eval_str(pymodule.get_scope(), first).get_object().source_function + is first_function.source_function + ) + # Repeat in the opposite order on the same module and keep both views alive. + reversed_second = evaluate.eval_str(pymodule.get_scope(), second) + reversed_first = evaluate.eval_str(pymodule.get_scope(), first) + assert ( + reversed_second.get_object().source_function is second_function.source_function + ) + assert reversed_first.get_object().source_function is first_function.source_function + assert second_view.get_object() is second_function + field_value = "1" if renamed_owner == "Owner" else "2" + offset = source.index("self.field = " + field_value) + len("self.") + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + after = {} + exec(compile(resource.read(), "storage_fixture.py", "exec"), after) + assert after["observed"] == before["observed"] + assert ("observed = result.changed" in resource.read()) == ( + renamed_owner == expected_owner + ) + + +def runtime_observed(source): + namespace = {} + exec(compile(source, "source_contract.py", "exec", dont_inherit=True), namespace) + return namespace["observed"] + + +@pytest.mark.parametrize( + "factory", + [ + "lambda supplied: (lambda *, option=None: supplied)", + "lambda supplied: (lambda option=supplied: option)", + ], +) +@pytest.mark.parametrize("renamed", ["first_field", "second_field"]) +def test_lambda_creation_values_are_isolated_between_calls( + model_project, factory, renamed +): + source = ( + "class First:\n first_field = 11\n" + "class Second:\n second_field = 22\n" + f"factory = {factory}\n" + "first = factory(First())\nsecond = factory(Second())\n" + "observed = (first().first_field, second().second_field)\n" + ) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + first = evaluate.eval_str(pymodule.get_scope(), "first") + second = evaluate.eval_str(pymodule.get_scope(), "second") + assert first.get_object().source_function is second.get_object().source_function + assert first.get_object() is not second.get_object() + for expression, expected in [("second()", "Second"), ("first()", "First")]: + assert evaluate.eval_str( + pymodule.get_scope(), expression + ).get_object().get_type() is (pymodule[expected].get_object()) + inner = first.get_object().source_function + assert inner.get_parameters()["option"].get_object() is pyobjects.get_unknown() + model_project.pycore.module_cache.forget_all_data() + assert inner.get_parameters()["option"].get_object() is pyobjects.get_unknown() + before = runtime_observed(source) + model_project.do( + rename.Rename( + model_project, resource, source.index(renamed + " =") + ).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before == (11, 22) + other = "second_field" if renamed == "first_field" else "first_field" + assert f".{other}" in resource.read() + + +@pytest.mark.parametrize( + "expression", + [ + "[value for value in (item,)][0]", + "[second for first, second in ((None, item),)][0]", + "[second for first in (item,) for second in (first,)][0]", + "{key: value for key, value in ((0, item),)}[0]", + ], +) +@pytest.mark.parametrize("renamed_owner", ["Payload", "Other"]) +def test_comprehension_call_overlay_drives_field_rename( + model_project, expression, renamed_owner +): + source = ( + "class Payload:\n field = 42\n" + "class Other:\n field = 99\n" + f"callback = lambda item: {expression}\n" + "result = callback(Payload())\nobserved = result.field\n" + ) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + callback = pymodule["callback"].get_object() + parameter = callback.get_parameters()["item"] + assert ( + pymodule["result"].get_object().get_type() is pymodule["Payload"].get_object() + ) + assert parameter.get_object() is pyobjects.get_unknown() + model_project.pycore.module_cache.forget_all_data() + assert callback.get_parameters()["item"] is parameter + assert parameter.get_object() is pyobjects.get_unknown() + before = runtime_observed(source) + declaration = "field = 42" if renamed_owner == "Payload" else "field = 99" + model_project.do( + rename.Rename(model_project, resource, source.index(declaration)).get_changes( + "changed" + ) + ) + assert runtime_observed(resource.read()) == before == 42 + assert ("observed = result.changed" in resource.read()) == ( + renamed_owner == "Payload" + ) + + +@pytest.mark.parametrize( + "descriptor,body", + [ + ("lambda self, item: self", "Owner"), + ("staticmethod(lambda item: item)", "Other"), + ("classmethod(lambda cls, item: item)", "Other"), + ], +) +def test_saved_lambda_keyword_rename_keeps_descriptor_and_source_identity( + model_project, descriptor, body +): + source = ( + "class Owner:\n field = 1\n" + f" callback = {descriptor}\n" + "class Other:\n field = 2\n" + "saved = Owner().callback\nresult = saved(item=Other())\n" + "observed = result.field\n" + ) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("item=Other()") + function = pymodule["saved"].get_object() + parameter = function.get_parameters()["item"] + assert evaluate.eval_location(pymodule, offset) is parameter + assert codeassist.get_definition_location( + model_project, source, offset, resource + ) == (resource, 3) + assert pymodule["result"].get_object().get_type() is pymodule[body].get_object() + assert parameter.get_object() is pyobjects.get_unknown() + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert "saved(changed=Other())" in resource.read() + assert runtime_observed(resource.read()) == before + + +@pytest.mark.parametrize("kind", ["closure", "saved-bound"]) +@pytest.mark.parametrize("renamed_owner", ["Owner", "Other"]) +def test_saved_bound_lambda_is_not_bound_again(model_project, kind, renamed_owner): + source = ( + "class Owner:\n field = 1\n" + " callback = lambda self, item: " + + ("(lambda: self)\n" if kind == "closure" else "self\n") + + "class Other:\n field = 2\n" + ) + source += ( + "saved = Owner().callback\nresult = saved(Other())()\n" + if kind == "closure" + else "class Saved:\n callback = Owner().callback\nresult = Saved().callback(Other())\n" + ) + source += "observed = result.field\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + assert pymodule["result"].get_object().get_type() is pymodule["Owner"].get_object() + before = runtime_observed(source) + offset = source.index("field = " + ("1" if renamed_owner == "Owner" else "2")) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before == 1 + assert ("observed = result.changed" in resource.read()) == ( + renamed_owner == "Owner" + ) + + +@pytest.mark.parametrize("owner", ["module", "class", "global", "nonlocal-parameter"]) +def test_header_created_name_is_queryable_before_and_after_cache_reset( + model_project, owner +): + if owner == "module": + source = "def inner(value=(target := str)):\n return target\nobserved = inner() is str\n" + elif owner == "class": + source = "class Owner:\n def inner(value=(target := str)):\n return value\nobserved = Owner.target is str and Owner.inner() is str\n" + elif owner == "global": + source = "def owner():\n global target\n def inner(value=(target := str)):\n return target\n return inner\nobserved = owner()() is str and target is str\n" + else: + source = "def enclosing(target):\n def owner():\n nonlocal target\n def inner(value=(target := str)):\n return target\n return inner\n return owner()\nobserved = enclosing(int)() is str\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target :=") + binder = evaluate.eval_location(pymodule, offset) + assert binder is not None + location = codeassist.get_definition_location( + model_project, source, offset, resource + ) + assert location[0] == resource + model_project.pycore.module_cache.forget_all_data() + assert evaluate.eval_location(pymodule, offset) is binder + assert ( + codeassist.get_definition_location(model_project, source, offset, resource) + == location + ) + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before is True + assert "target" not in resource.read() + + +@pytest.mark.parametrize("selection", ["module", "lambda", "comprehension"]) +def test_nested_lambda_comprehension_shadowing_rename_preserves_each_binding( + model_project, selection +): + source = ( + "target = int\n" + "result = (lambda *, target: [target for target in target])(target=(str, bytes))\n" + "observed = (target is int, result == [str, bytes])\n" + ) + resource = module(model_project, source) + markers = { + "module": "target = int", + "lambda": "target:", + "comprehension": "for target", + } + offset = source.index(markers[selection]) + ( + 4 if selection == "comprehension" else 0 + ) + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before == (True, True) + if selection == "lambda": + assert "[target for target in changed]" in resource.read() + assert ")(changed=" in resource.read() + elif selection == "comprehension": + assert "[changed for changed in target]" in resource.read() + else: + assert "lambda *, target:" in resource.read() + + +@pytest.mark.parametrize( + "expression", + ["lambda: (target := str)", "lambda: [(target := value) for value in [str]][0]"], +) +@pytest.mark.parametrize("operation", ["local", "outer", "occurrences"]) +def test_lambda_walrus_source_binding_does_not_escape_to_module( + model_project, expression, operation +): + source = f"target = int\ncallback = {expression}\nobserved = (target is int, callback() is str)\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + local_offset = source.index("target :=") + local = evaluate.eval_location(pymodule, local_offset) + outer = pymodule["target"] + assert local is not outer + assert local.get_definition_location() == (pymodule, 2) + before = runtime_observed(source) + if operation == "occurrences": + changes = rename.ChangeOccurrences( + model_project, resource, local_offset + ).get_changes("changed") + else: + offset = local_offset if operation == "local" else 0 + changes = rename.Rename(model_project, resource, offset).get_changes("changed") + model_project.do(changes) + assert runtime_observed(resource.read()) == before == (True, True) + assert ("target = int" in resource.read()) == (operation != "outer") + assert ("target :=" in resource.read()) == (operation == "outer") + + +@pytest.mark.parametrize("mode", ["eager", "future", "lazy"]) +@pytest.mark.parametrize( + "class_body,selected", + [ + ("target = bytes\n", "class"), + ("target: object = bytes\n", "class"), + ("target: object\n", "module"), + ("target = bytes\ndel target\n", "module"), + ("pass\n", "module"), + ], +) +def test_annotation_query_completion_and_rename_follow_class_namespace( + model_project, mode, class_body, selected +): + source = prefix(mode) + "target = int\nclass Owner:\n" + source += indent( + class_body + "def func(value: target) -> target:\n return value\n", " " + ) + source += "subject = Owner.func\ndef drive():\n return subject(str)\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target", source.index("value: ")) + # Future-string get_type_hints uses the module namespace by its default contract. + expected = pymodule["target"] + if mode != "future" and selected == "class": + expected = pymodule["Owner"].get_object()["target"] + context = pymodule.get_source_binding_index().context_at(offset) + assert evaluate.eval_location(pymodule, offset) is expected + assert context.structural_scope.get_kind() == "Function" + assert context.environment.get_kind() == "Annotation" + assert ( + context.annotation_policy.mode + == {"eager": "eager", "future": "future-string", "lazy": "lazy"}[mode] + ) + assert codeassist.get_definition_location( + model_project, source, offset, resource + ) == (resource, expected.get_definition_location()[1]) + proposals = codeassist.code_assist( + model_project, source, offset + 3, resource=resource, later_locals=False + ) + assert "target" in {proposal.name for proposal in proposals} + model_project.pycore.module_cache.forget_all_data() + assert evaluate.eval_location(pymodule, offset) is expected + before = observe(source) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert observe(resource.read()) == before + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="type aliases require Python 3.12" +) +@pytest.mark.parametrize("owner", ["module", "class", "class-fallback", "nested"]) +def test_type_alias_outer_reference_rename_preserves_lazy_value(model_project, owner): + if owner == "module": + source = "target = int\ntype Alias = tuple[target, target]\nsubject = Alias\n" + elif owner == "class": + source = "target = int\nclass Owner:\n target = bytes\n type Alias = tuple[target, target]\nsubject = Owner.Alias\n" + elif owner == "class-fallback": + source = "target = int\nclass Owner:\n type Alias = tuple[target, target]\nsubject = Owner.Alias\n" + else: + source = "target = int\ndef factory():\n target = bytes\n type Alias = tuple[target, target]\n return Alias\nsubject = factory()\n" + source += "observed = tuple(item.__name__ for item in subject.__value__.__args__)\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target", source.index("tuple[")) + binding = evaluate.eval_location(pymodule, offset) + assert binding is not None and getattr( + binding, "is_refactoring_identity_safe", True + ) + assert ( + codeassist.get_definition_location(model_project, source, offset, resource)[0] + == resource + ) + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="generic classes require Python 3.12" +) +@pytest.mark.parametrize("query_first", [False, True]) +def test_generic_alias_queries_remain_readable_and_refusal_is_project_atomic( + model_project, query_first +): + source = "target = int\nclass Risk[target]:\n target = str\n type Alias = tuple[target, target]\n" + resource = module(model_project, source) + client = model_project.root.create_file("client.py") + client.write("from subject import target\nobserved = target is int\n") + pymodule = model_project.get_pymodule(resource) + offset = source.index("target", source.index("tuple[")) + before = {file: file.read() for file in (resource, client)} + if query_first: + assert codeassist.get_definition_location( + model_project, source, offset, resource + ) == (resource, 4) + assert finderrors.find_errors(model_project, resource) == [] + proposals = codeassist.code_assist( + model_project, source, offset + 3, resource=resource + ) + assert "target" in {proposal.name for proposal in proposals} + opaque = evaluate.eval_location(pymodule, offset) + assert opaque.get_object() is pyobjects.get_unknown() + assert not opaque.is_refactoring_identity_safe + for selected in [ + offset, + source.index("[target]") + 1, + source.index("target = str"), + ]: + with pytest.raises(exceptions.RefactoringError, match="unresolved"): + rename.Rename(model_project, resource, selected).get_changes("changed") + assert {file: file.read() for file in before} == before + # The same module's unrelated exported source name remains independently editable. + model_project.do(rename.Rename(model_project, resource, 0).get_changes("changed")) + assert "from subject import changed" in client.read() + assert "type Alias = tuple[target, target]" in resource.read() + + +def test_change_occurrences_checks_only_the_selected_structural_region(model_project): + source = ( + "target = int\nenabled = True\nclass Risk:\n if enabled:\n target = str\n" + " def method(value: target):\n return value\n" + "def allowed():\n return target\nobserved = allowed() is int\n" + ) + resource = module(model_project, source) + before = runtime_observed(source) + offset = source.index("target", source.index("return ")) + model_project.do( + rename.ChangeOccurrences(model_project, resource, offset).get_changes("int") + ) + assert runtime_observed(resource.read()) == before is True + assert "return int" in resource.read() + assert "value: target" in resource.read() + resource.write(source) + with pytest.raises(exceptions.RefactoringError, match="unresolved"): + rename.ChangeOccurrences(model_project, resource, 0).get_changes("changed") + assert resource.read() == source + + +@pytest.mark.parametrize( + "source,entry_marker,body_marker", + [ + ( + "lex_outer = int\ncallback = lambda lex_item=lex_outer: lex_item\n", + "=lex_outer", + ": lex_item", + ), + ( + "lex_outer = [int]\nvalues = [lex_item for lex_item in lex_outer]\n", + "in lex_outer", + "[lex_item", + ), + ], +) +def test_completion_separates_expression_entry_from_local_body( + model_project, source, entry_marker, body_marker +): + resource = module(model_project, source) + for marker, body in [(entry_marker, False), (body_marker, True)]: + start = source.index("lex_", source.index(marker)) + proposals = { + item.name: item.scope + for item in codeassist.code_assist( + model_project, source, start + 4, resource=resource, later_locals=False + ) + } + assert proposals["lex_outer"] == "global" + if body: + assert proposals["lex_item"] == "local" + else: + assert "lex_item" not in proposals + + +@pytest.mark.parametrize( + "source,expected", + [ + ("callback = lambda item: item\n", []), + ("values = [item for item in [int]]\n", []), + ( + "callback = lambda item: (item, absent)\n", + [(1, "Unresolved variable absent")], + ), + ( + "values = [item for item in [int] if absent]\n", + [(1, "Unresolved variable absent")], + ), + ("result = later\nlater = int\n", [(1, "Defined later later")]), + ], +) +def test_find_errors_preserves_real_diagnostics_without_expression_false_positives( + model_project, source, expected +): + resource = module(model_project, source) + assert [ + (error.lineno, error.error) + for error in finderrors.find_errors(model_project, resource) + ] == expected + + +def test_introduce_parameter_rewrites_outer_capture_in_comprehension(model_project): + source = "value = int\ndef subject():\n return [value for item in [str]]\nobserved = subject() == [int]\n" + resource = module(model_project, source) + before = runtime_observed(source) + operation = introduce_parameter.IntroduceParameter( + model_project, resource, source.index("value", source.index("return ")) + ) + model_project.do(operation.get_changes("provided")) + assert runtime_observed(resource.read()) == before is True + assert "def subject(provided=value)" in resource.read() + assert "[provided for item" in resource.read() + assert ( + runtime_observed(resource.read() + "observed = subject(str) == [str]\n") is True + ) + resource.write(source) + with pytest.raises(exceptions.RefactoringError): + introduce_parameter.IntroduceParameter( + model_project, resource, source.index("item in") + ) + assert resource.read() == source + + +@pytest.mark.parametrize("future", [False, True]) +def test_introduce_parameter_refuses_annotation_environment_atomically( + model_project, future +): + source = "from __future__ import annotations\n" if future else "" + source += "value = int\ndef subject(item: value):\n return item\n" + resource = module(model_project, source) + offset = source.index("value", source.index("item:")) + with pytest.raises(exceptions.RefactoringError, match="inside functions"): + introduce_parameter.IntroduceParameter(model_project, resource, offset) + assert resource.read() == source + + +@pytest.mark.parametrize("renamed_owner", ["Owner", "Other"]) +def test_callable_instance_lambda_drives_field_rename(model_project, renamed_owner): + source = "class Owner:\n field = 1\n __call__ = lambda self, item: self\nclass Other:\n field = 2\nresult = Owner()(Other())\nobserved = result.field\n" + resource = module(model_project, source) + before = runtime_observed(source) + offset = source.index("field = " + ("1" if renamed_owner == "Owner" else "2")) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before == 1 + assert ("observed = result.changed" in resource.read()) == ( + renamed_owner == "Owner" + ) + + +def test_callable_instance_lambda_keyword_has_canonical_formal_identity(model_project): + source = "class Owner:\n __call__ = lambda self, item: item\nresult = Owner()(item=str)\nobserved = result is str\n" + resource = module(model_project, source) + offset = source.index("item=str") + pymodule = model_project.get_pymodule(resource) + binder = ( + pymodule["Owner"].get_object()["__call__"].get_object().get_parameters()["item"] + ) + assert evaluate.eval_location(pymodule, offset) is binder + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) is True + assert "Owner()(changed=str)" in resource.read() + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="type aliases require Python 3.12" +) +def test_class_type_alias_captures_generic_owner_without_capturing_class_values( + model_project, +): + source = "T = int\nclass Owner[T]:\n unrelated = bytes\n type Alias = tuple[T, T]\nobserved = Owner.Alias.__value__.__args__ == (Owner.__type_params__[0],) * 2\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("T", source.index("type Alias")) + binder = evaluate.eval_location(pymodule, offset) + own = evaluate.eval_location(pymodule, source.index("[T]") + 1) + assert binder is own and binder is not pymodule["T"] + proposals = codeassist.code_assist( + model_project, source, offset + 1, resource=resource + ) + assert "T" in {proposal.name for proposal in proposals} + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, source.index("[T]") + 1).get_changes("V") + ) + assert runtime_observed(resource.read()) == before is True + assert "T = int" in resource.read() + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="generic functions require Python 3.12" +) +@pytest.mark.parametrize("expression", ["T", "(lambda: T)()"]) +def test_generic_annotation_query_remains_opaque_through_nested_lambda( + model_project, expression +): + source = f"T = int\ndef subject[T](value: {expression}):\n return value\nobserved = subject(str) is str\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("T", source.index("value: ")) + binder = evaluate.eval_location(pymodule, offset) + assert not binder.is_refactoring_identity_safe + assert binder.get_object() is pyobjects.get_unknown() + assert ( + codeassist.get_definition_location(model_project, source, offset, resource)[0] + == resource + ) + proposals = codeassist.code_assist( + model_project, source, offset + 1, resource=resource + ) + assert "T" in {proposal.name for proposal in proposals} + assert runtime_observed(source) is True + for selected in [offset, source.index("[T]") + 1]: + with pytest.raises(exceptions.RefactoringError, match="unresolved"): + rename.Rename(model_project, resource, selected).get_changes("V") + assert resource.read() == source + + +@pytest.mark.skipif( + sys.version_info >= (3, 14), reason="annotation walrus requires eager annotations" +) +@pytest.mark.parametrize("owner", ["module", "class"]) +def test_eager_annotation_walrus_installs_its_declaration_binding(model_project, owner): + if owner == "module": + source = "def subject(value: (target := str)):\n return value\nobserved = subject(int) is int and target is str\n" + else: + source = "target = int\nclass Owner:\n def subject(value: (target := str)):\n return value\nobserved = Owner.subject(int) is int and Owner.target is str and target is int\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target :=") + binding = evaluate.eval_location(pymodule, offset) + if owner == "class": + assert binding is pymodule["Owner"].get_object()["target"] + assert binding is not pymodule["target"] + else: + assert binding is pymodule["target"] + assert binding.get_definition_location()[0] is pymodule + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before is True + + +@pytest.mark.parametrize("selection", ["outer", "class"]) +def test_class_lambda_default_walrus_does_not_change_outer_source_binding( + model_project, selection +): + source = "target = int\nclass Owner:\n callback = staticmethod(lambda value=(target := str): value)\nobserved = Owner.callback() is str and Owner.target is str and target is int\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target :=") if selection == "class" else 0 + selected = evaluate.eval_location(pymodule, offset) + class_binding = evaluate.eval_location(pymodule, source.index("target :=")) + assert class_binding is not pymodule["target"] + assert selected is (class_binding if selection == "class" else pymodule["target"]) + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before is True + + +@pytest.mark.parametrize("mode", ["eager", "future", "lazy"]) +@pytest.mark.parametrize("declaration", ["import", "function", "class", "late"]) +def test_class_annotation_namespace_obeys_declaration_order_and_name_shadowing( + model_project, mode, declaration +): + source = prefix(mode) + "import typing\ntarget = int\nclass Owner:\n" + if declaration == "import": + body = "from builtins import str as target\n" + expression = "typing.get_type_hints(Owner.subject)['value'] is str" + elif declaration == "function": + body = "def target():\n return bytes\n" + expression = "typing.get_type_hints(Owner.subject)['value']() is bytes" + elif declaration == "class": + body = "class target:\n marker = 7\n" + expression = "typing.get_type_hints(Owner.subject)['value'].marker == 7" + else: + body = "" + expression = ( + "typing.get_type_hints(Owner.subject)['value'] is bytes" + if mode == "lazy" + else "typing.get_type_hints(Owner.subject)['value'] is int" + ) + body += "def subject(value: target):\n return value\n" + if declaration == "late": + body += "target = bytes\n" + if mode == "future": + expression = "typing.get_type_hints(Owner.subject)['value'] is int" + source += indent(body, " ") + source += f"observed = (Owner.subject(str) is str, {expression}, target is int)\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target", source.index("value: ")) + expected = pymodule["target"] + class_resolver = mode != "future" and (declaration != "late" or mode == "lazy") + if class_resolver: + expected = pymodule["Owner"].get_object()["target"] + assert evaluate.eval_location(pymodule, offset) is expected + proposals = codeassist.code_assist( + model_project, source, offset + 3, resource=resource, later_locals=False + ) + assert "target" in {proposal.name for proposal in proposals} + before = runtime_observed(source) + # Renaming the outer source must leave independently declared class names intact. + model_project.do( + rename.Rename( + model_project, resource, source.index("target = int") + ).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before == (True, True, True) + assert ("value: target" in resource.read()) == class_resolver + + +@pytest.mark.parametrize( + "expression", ["lambda item: item", "[item for item in [str]]"] +) +def test_change_occurrences_limits_edits_to_expression_binding( + model_project, expression +): + if "lambda" in expression: + source = f"item = int\ncallback = {expression}\nobserved = (callback(str) is str, item is int)\n" + offset = source.index("item:") + else: + source = f"item = int\nresult = {expression}\nobserved = (result == [str], item is int)\n" + offset = source.index("item", source.index("for ")) + resource = module(model_project, source) + before = runtime_observed(source) + model_project.do( + rename.ChangeOccurrences(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before == (True, True) + assert "item = int" in resource.read() + assert "item is int" in resource.read() + + +@pytest.mark.parametrize( + "expression", + [ + "[item for item in owners][0]", + "{0: item for item in owners}[0]", + "{item for item in owners}.pop()", + "tuple(item for item in owners)[0]", + "[right for left, right in ((other, owner),)][0]", + "(lambda ignored: owner)(other)", + ], +) +def test_independently_parsed_expression_queries_keep_legacy_value_scope( + model_project, expression +): + source = ( + "class Owner:\n field = 1\nclass Other:\n field = 2\n" + "owner = Owner()\nother = Other()\nitem = other\nowners = [owner]\n" + ) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + namespace = {} + exec(compile(source, "query_source.py", "exec", dont_inherit=True), namespace) + result = eval(expression, namespace) + assert type(result) is namespace["Owner"] and result.field == 1 + item_offset = source.index("item = other") + source_binding = evaluate.eval_location(pymodule, item_offset) + assert source_binding.get_object().get_type() is pymodule["Other"].get_object() + for cold in (False, True): + if cold: + model_project.pycore.module_cache.forget_all_data() + inferred = evaluate.eval_str(pymodule.get_scope(), expression) + _, inferred_pair = evaluate.eval_str2(pymodule.get_scope(), expression) + assert inferred.get_object().get_type() is pymodule["Owner"].get_object() + assert inferred_pair.get_object().get_type() is pymodule["Owner"].get_object() + assert evaluate.eval_location(pymodule, item_offset) is source_binding + assert source_binding.get_object().get_type() is pymodule["Other"].get_object() + assert resource.read() == source + assert namespace["item"] is namespace["other"] + + +def test_public_binding_tree_locates_local_region_and_preserves_rename_driver( + model_project, +): + source = ( + "item = int\nouter_only = bytes\ncallback = (\n" + " lambda item: [\n item\n for item in item\n ]\n)\n" + "observed = (callback([str, bytes]) == [str, bytes], item is int)\n" + ) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + + def locate(module_, code, name): + # A client traverses binding environments separately from structural scopes. + (lambda_scope,) = module_.get_scope().get_binding_scopes() + (comp_scope,) = lambda_scope.get_binding_scopes() + assert lambda_scope.get_kind() == "Lambda" + assert comp_scope.get_kind() == "Comprehension" + assert "outer_only" not in lambda_scope.get_names() + assert "outer_only" not in comp_scope.get_names() + formal = lambda_scope["item"] + local = comp_scope[name] + assert local is not formal and formal is not module_["item"] + lambda_start, lambda_end = lambda_scope.get_region() + comp_start, comp_end = comp_scope.get_region() + assert lambda_start < comp_start < comp_end <= lambda_end + assert code[lambda_start:lambda_end].startswith("lambda item:") + assert code[comp_start:comp_end].startswith("[") + declaration = code.index(name, code.index("for ")) + body = code.index(name, comp_start) + iterable = code.index("item", code.index("in item")) + assert evaluate.eval_location(module_, declaration) is local + assert evaluate.eval_location(module_, body) is local + assert evaluate.eval_location(module_, iterable) is formal + assert local.get_definition_location() == (module_, 6) + assert codeassist.get_definition_location( + model_project, code, body, resource + ) == (resource, 6) + return lambda_scope, comp_scope, local, declaration + + lambda_scope, comp_scope, binding, declaration = locate(pymodule, source, "item") + model_project.pycore.module_cache.forget_all_data() + cold_lambda, cold_comp, cold_binding, _ = locate(pymodule, source, "item") + assert cold_lambda is lambda_scope and cold_comp is comp_scope + assert cold_binding is binding + before = runtime_observed(source) + model_project.do( + rename.Rename(model_project, resource, declaration).get_changes("changed") + ) + after = resource.read() + assert runtime_observed(after) == before == (True, True) + assert "for changed in item" in after + assert "lambda item:" in after and "item = int" in after + updated = model_project.get_pymodule(resource) + _, _, renamed_binding, _ = locate(updated, after, "changed") + assert renamed_binding is not updated["item"] + + +def test_calltip_reads_simple_source_lambda_without_changing_its_binding(model_project): + source = "callback = lambda item: item\nobserved = callback(str) is str\n" + resource = module(model_project, source) + offset = source.index("callback(str)") + 1 + assert ( + codeassist.get_calltip(model_project, source, offset, resource=resource) + == "lambda(item)" + ) + assert runtime_observed(source) is True + assert resource.read() == source + + +@pytest.mark.parametrize("renamed_owner", ["Owner", "Other"]) +def test_lambda_walrus_local_value_keeps_outer_value_and_field_rename_independent( + model_project, renamed_owner +): + source = ( + "class Owner:\n field = 1\nclass Other:\n field = 2\n" + "local = Other()\nresult = (lambda: ((local := Owner()), local)[1])()\n" + "observed = (result.field, local.field)\n" + ) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + local = evaluate.eval_location(pymodule, source.index("local :=")) + assert local is not pymodule["local"] + assert local.get_object().get_type() is pymodule["Owner"].get_object() + assert pymodule["local"].get_object().get_type() is pymodule["Other"].get_object() + assert pymodule["result"].get_object().get_type() is pymodule["Owner"].get_object() + before = runtime_observed(source) + offset = source.index("field = " + ("1" if renamed_owner == "Owner" else "2")) + model_project.do( + rename.Rename(model_project, resource, offset).get_changes("changed") + ) + assert runtime_observed(resource.read()) == before == (1, 2) + assert ("result.changed" in resource.read()) == (renamed_owner == "Owner") + assert ("local.changed" in resource.read()) == (renamed_owner == "Other") + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="type aliases require Python 3.12" +) +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.parametrize("query_first", [False, True]) +@pytest.mark.parametrize("refactoring", [rename.Rename, rename.ChangeOccurrences]) +def test_conditional_class_alias_queries_are_readable_and_candidate_edits_are_atomic( + model_project, enabled, query_first, refactoring +): + source = ( + f"target = int\nenabled = {enabled}\nclass Owner:\n" + " if enabled:\n target = bytes\n" + " type Alias = tuple[target, target]\n" + "unrelated = 42\nobserved = unrelated\n" + ) + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target", source.index("tuple[")) + if not query_first: + with pytest.raises(exceptions.RefactoringError, match="unresolved"): + refactoring(model_project, resource, 0).get_changes("changed") + assert ( + evaluate.eval_location(pymodule, offset).get_object() is pyobjects.get_unknown() + ) + assert codeassist.get_definition_location( + model_project, source, offset, resource + ) == (resource, 6) + assert finderrors.find_errors(model_project, resource) == [] + before = {} + exec(compile(source, "conditional_alias.py", "exec", dont_inherit=True), before) + expected = bytes if enabled else int + assert before["Owner"].Alias.__value__.__args__ == (expected, expected) + for selection in (offset, 0, source.index("target = bytes")): + with pytest.raises(exceptions.RefactoringError, match="unresolved"): + refactoring(model_project, resource, selection).get_changes("changed") + assert resource.read() == source + model_project.do( + rename.Rename(model_project, resource, source.index("unrelated =")).get_changes( + "safe" + ) + ) + after = {} + exec( + compile(resource.read(), "conditional_alias.py", "exec", dont_inherit=True), + after, + ) + assert after["Owner"].Alias.__value__.__args__ == (expected, expected) + assert after["observed"] == before["observed"] == 42 + + +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="type parameters require Python 3.12" +) +def test_generic_class_bound_query_and_outer_rename_keep_runtime_bound(model_project): + source = "target = int\nclass Owner[T: target]:\n pass\nobserved = Owner.__type_params__[0].__bound__ is int\n" + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + offset = source.index("target", source.index("[T:")) + assert evaluate.eval_location(pymodule, offset) is pymodule["target"] + assert codeassist.get_definition_location( + model_project, source, offset, resource + ) == (resource, 1) + before = runtime_observed(source) + model_project.do(rename.Rename(model_project, resource, 0).get_changes("changed")) + assert runtime_observed(resource.read()) == before is True + assert "class Owner[T: changed]" in resource.read() + + +def test_foreign_node_has_no_context_and_reads_no_source(model_project): + source = 'nome = "ação"\nvalor: int = 1\n' + resource = module(model_project, source) + pymodule = model_project.get_pymodule(resource) + index = pymodule.get_source_binding_index() + # A consumer's own parse of another text: its end column (10) falls + # inside the two-byte "ç" of this module's first line. + foreign = ast.parse("valorxxxxx\n").body[0].value + assert (foreign.lineno, foreign.end_col_offset) == (1, 10) + assert index.context_for_node(foreign) is None + assert evaluate.eval_node(pymodule.get_scope(), foreign) is None + owned = pymodule.get_ast().body[1].annotation + assert index.context_for_node(owned) is not None diff --git a/ropetest/testutils.py b/ropetest/testutils.py index 11f58ccfe..94befee1e 100644 --- a/ropetest/testutils.py +++ b/ropetest/testutils.py @@ -98,15 +98,3 @@ def only_for_versions_higher(version): def skipNotPOSIX(): return unittest.skipIf(os.name != "posix", "This test works only on POSIX") - - -def time_limit(timeout): - if not any(procname in sys.argv[0] for procname in {"pytest", "py.test"}): - # no-op when running tests without pytest - return lambda *args, **kwargs: lambda func: func - - # do a local import so we don't import pytest when running without pytest - import pytest - - # this prevents infinite loop/recursion from taking forever in CI - return pytest.mark.time_limit(timeout) diff --git a/ropetest/type_hinting_test.py b/ropetest/type_hinting_test.py index af94352fb..9cf322acd 100644 --- a/ropetest/type_hinting_test.py +++ b/ropetest/type_hinting_test.py @@ -1,9 +1,9 @@ +import sys import unittest from textwrap import dedent, indent -import pytest - -from rope.base.oi.type_hinting import evaluate +from rope.base import builtins, pyobjects +from rope.base.oi.type_hinting import evaluate, utils from rope.contrib.codeassist import code_assist from ropetest import testutils @@ -42,6 +42,729 @@ def run(self, result=None): super().run(result) +class UnionAlternativeHintingTest(AbstractHintingTest): + def test_unresolved_alternative_keeps_the_complete_union_unresolved(self): + resource = testutils.create_module(self.project, "union_hint") + resource.write("import types\nclass Known:\n marker = 1\n") + module = self.project.get_pymodule(resource) + for hint in ("Known | types.Missing", "types.Missing | Known", "Known or types.Missing"): + with self.subTest(hint=hint): + self.assertIsNone(evaluate.evaluate(hint, module)) + + def test_resolved_none_type_remains_a_native_union_alternative(self): + resource = testutils.create_module(self.project, "union_hint") + resource.write("import types\nclass Known:\n marker = 1\n") + module = self.project.get_pymodule(resource) + none_type = utils.resolve_type("types.NoneType", module) + self.assertIsNotNone(none_type) + self.assertTrue(builtins._is_runtime_class(none_type, type(None))) + result = evaluate.evaluate("Known | types.NoneType", module) + self.assertIs(result.alternatives[1], none_type) + self.assertIn("marker", result.get_attributes()) + + +class SourceAnnotationHintingTest(AbstractHintingTest): + def test_unresolved_annotation_is_the_unknown_sentinel(self): + resource = testutils.create_module(self.project, 'unresolved_annotation') + resource.write("def choose(value: Missing) -> Missing:\n return value\n") + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertIs(pyobjects.get_unknown(), function.get_parameters()['value'].get_object()) + self.assertIs(pyobjects.get_unknown(), function.get_returned_object(None)) + + def test_sequence_invalid_arity_is_not_a_guessed_element_contract(self): + code = ( + "from collections.abc import Sequence\n" + "def choose(items: Sequence[str, int]):\n return items[0]\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual('real', namespace['choose'](['real'])) + resource = testutils.create_module(self.project, 'invalid_sequence_arity') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertEqual(pyobjects.get_unknown(), function.get_parameters()['items'].get_object()) + + def test_mutable_sequence_is_not_claimed_as_read_only_sequence(self): + code = ( + "from typing import MutableSequence\n" + "def choose(items: MutableSequence[str]):\n items.append('real')\n" + ) + namespace = {} + exec(code, namespace) + values = [] + namespace['choose'](values) + self.assertEqual(['real'], values) + resource = testutils.create_module(self.project, 'mutable_sequence') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertEqual(pyobjects.get_unknown(), function.get_parameters()['items'].get_object()) + + def test_sequence_tuple_annotation_preserves_element_positions(self): + for origin in ('collections.abc', 'typing'): + with self.subTest(origin=origin): + code = ( + f"from {origin} import Sequence\n" + "class Row:\n" + " @property\n def label(self) -> str:\n return 'real'\n" + "def choose(items: Sequence[tuple[str, Row]]):\n" + " for _, row in items:\n return row.label\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual('real', namespace['choose']([('key', namespace['Row']())])) + resource = testutils.create_module(self.project, 'sequence_' + origin.replace('.', '_')) + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + returned = function.get_returned_object(None) + self.assertIsNotNone(returned) + self.assertIs(builtins.get_str_type(), returned.get_type()) + + def test_sequence_slicing_preserves_read_only_element_contract(self): + code = ( + "from collections.abc import Sequence\n" + "class Row:\n" + " @property\n def label(self) -> str:\n return 'real'\n" + "def choose(items: Sequence[Row]):\n return items[:1][0].label\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual('real', namespace['choose']([namespace['Row']()])) + resource = testutils.create_module(self.project, 'sequence_slice') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + parameter = function.get_parameters()['items'].get_object() + self.assertIn('__getitem__', parameter) + self.assertNotIn('append', parameter) + self.assertNotIn('__setitem__', parameter) + returned = function.get_returned_object(None) + self.assertIsNotNone(returned) + self.assertIs(builtins.get_str_type(), returned.get_type()) + + def test_fixed_tuple_annotation_does_not_collapse_to_first_element(self): + code = "def choose(items: tuple[str, int]):\n return items[1]\n" + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['choose'](('key', 7))) + resource = testutils.create_module(self.project, 'tuple_positions') + resource.write(code) + returned = self.project.get_pymodule(resource)['choose'].get_object().get_returned_object(None) + self.assertIs(builtins.builtins['int'].get_object(), returned.get_type()) + + def test_local_sequence_keeps_its_actual_iteration_contract(self): + code = ( + "class Sequence:\n" + " @classmethod\n def __class_getitem__(cls, item):\n return cls\n" + " def __iter__(self):\n return iter([7])\n" + "def choose(items: Sequence[str]):\n return next(iter(items))\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['choose'](namespace['Sequence']())) + resource = testutils.create_module(self.project, 'local_sequence') + resource.write(code) + returned = self.project.get_pymodule(resource)['choose'].get_object().get_returned_object(None) + self.assertIs(builtins.builtins['int'].get_object(), returned.get_type()) + + def test_class_annotation_uses_lexical_sequence_binding(self): + for declaration, annotation in ( + ("Sequence = Local", "Sequence[str]"), + ("class types:\n Sequence = Local", "types.Sequence[str]"), + ): + with self.subTest(annotation=annotation): + code = ( + "from typing import Sequence\n" + "import typing as types\n" + "class Local:\n" + " @classmethod\n def __class_getitem__(cls, item):\n return cls\n" + " def __getitem__(self, key):\n return 7\n" + "class Owner:\n" + f" {declaration}\n" + f" def choose(self, items: {annotation}):\n return items[0]\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['Owner']().choose(namespace['Local']())) + resource = testutils.create_module(self.project, 'lexical_sequence_' + annotation.replace('.', '_').replace('[', '_').replace(']', '')) + resource.write(code) + module = self.project.get_pymodule(resource) + function = module['Owner'].get_object()['choose'].get_object() + parameter = function.get_parameters()['items'].get_object() + self.assertIs(module['Local'].get_object(), parameter.get_type()) + self.assertIs(builtins.builtins['int'].get_object(), function.get_returned_object(None).get_type()) + + def test_fixed_tuple_dynamic_index_does_not_invent_first_type(self): + code = "def choose(items: tuple[str, int], index: int):\n return items[index]\n" + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['choose'](('key', 7), 1)) + resource = testutils.create_module(self.project, 'tuple_dynamic_index') + resource.write(code) + returned = self.project.get_pymodule(resource)['choose'].get_object().get_returned_object(None) + self.assertEqual(pyobjects.get_unknown(), returned) + + def test_fixed_tuple_explicit_getitem_does_not_invent_first_type(self): + code = "def choose(items: tuple[str, int], index: int):\n return items.__getitem__(index)\n" + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['choose'](('key', 7), 1)) + resource = testutils.create_module(self.project, 'tuple_explicit_getitem') + resource.write(code) + returned = self.project.get_pymodule(resource)['choose'].get_object().get_returned_object(None) + self.assertEqual(pyobjects.get_unknown(), returned) + + def test_fixed_tuple_slices_preserve_positions_or_remain_unknown(self): + cases = (("[1:][0]", 7, 'int'), ("[::-1][0]", 7, 'int'), ("[:1][0]", 'key', 'str'), ("[index:][0]", 7, None)) + for number, (expression, expected, inferred) in enumerate(cases): + with self.subTest(expression=expression): + code = f"def choose(items: tuple[str, int], index: int):\n return items{expression}\n" + namespace = {} + exec(code, namespace) + self.assertEqual(expected, namespace['choose'](('key', 7), 1)) + resource = testutils.create_module(self.project, 'tuple_slice_' + str(number)) + resource.write(code) + returned = self.project.get_pymodule(resource)['choose'].get_object().get_returned_object(None) + if inferred is None: + self.assertEqual(pyobjects.get_unknown(), returned) + else: + self.assertIs(builtins.builtins[inferred].get_object(), returned.get_type()) + + def test_eager_annotation_preserves_declaration_epoch(self): + for number, code in enumerate(( + "Type = int\ndef choose(value: Type):\n return value\nType = str\n", + "Other = int\nType = Other\ndef choose(value: Type):\n return value\nOther = str\n", + "class Owner:\n Type = int\n def choose(self, value: Type):\n return value\n Type = str\n", + )): + with self.subTest(number=number): + namespace = {} + exec(code, namespace) + function = namespace['choose'] if 'choose' in namespace else namespace['Owner'].choose + # Eager before 3.14; PEP 649 evaluates after the module ran. + actual = function.__annotations__['value'] + self.assertIs(int if sys.version_info < (3, 14) or number == 1 else str, actual) + expected = {int: builtins.builtins['int'].get_object(), str: builtins.get_str_type()}[actual] + resource = testutils.create_module(self.project, 'eager_annotation_' + str(number)) + resource.write(code) + module = self.project.get_pymodule(resource) + function = module['choose'].get_object() if 'choose' in module else module['Owner'].get_object()['choose'].get_object() + self.assertIs(expected, function.get_parameters()['value'].get_object().get_type()) + + def test_postponed_annotation_keeps_late_binding(self): + code = "from __future__ import annotations\nType = int\ndef choose(value: Type):\n return value\nType = str\n" + namespace = {} + exec(code, namespace) + import typing + self.assertIs(str, typing.get_type_hints(namespace['choose'], namespace)['value']) + resource = testutils.create_module(self.project, 'postponed_annotation') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertIs(builtins.get_str_type(), function.get_parameters()['value'].get_object().get_type()) + + def test_string_annotation_keeps_late_binding(self): + code = "Type = int\ndef choose(value: 'Type'):\n return value\nType = str\n" + namespace = {} + exec(code, namespace) + import typing + self.assertIs(str, typing.get_type_hints(namespace['choose'], namespace)['value']) + resource = testutils.create_module(self.project, 'string_annotation') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertIs(builtins.get_str_type(), function.get_parameters()['value'].get_object().get_type()) + + def test_shadowed_annotated_is_not_unwrapped_as_stdlib(self): + code = ( + "from typing import Annotated\n" + "class Local:\n" + " @classmethod\n def __class_getitem__(cls, item):\n return cls\n" + "class Owner:\n Annotated = Local\n" + " def choose(self, value: Annotated[str, 'label']):\n return value\n" + ) + namespace = {} + exec(code, namespace) + self.assertIs(namespace['Local'], namespace['Owner'].choose.__annotations__['value']) + resource = testutils.create_module(self.project, 'shadowed_annotated') + resource.write(code) + module = self.project.get_pymodule(resource) + function = module['Owner'].get_object()['choose'].get_object() + self.assertIs(module['Local'].get_object(), function.get_parameters()['value'].get_object().get_type()) + + def test_typing_wrappers_do_not_become_function_parameter_types(self): + for number, (hint, expected) in enumerate((('Annotated[str, "label"]', 'str'), ('Optional[str]', None))): + with self.subTest(hint=hint): + code = f"from typing import Annotated, Optional\ndef choose(value: {hint}):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual('real', namespace['choose']('real')) + resource = testutils.create_module(self.project, 'typing_wrapper_' + str(number)) + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + parameter = function.get_parameters()['value'].get_object() + if expected: + self.assertIs(builtins.get_str_type(), parameter.get_type()) + else: + self.assertFalse(isinstance(parameter, pyobjects.AbstractFunction)) + self.assertFalse(isinstance(parameter.get_type(), pyobjects.AbstractFunction)) + if parameter != pyobjects.get_unknown(): + self.assertIsInstance(parameter.get_type(), evaluate.UnionType) + + def test_nested_forward_references_use_late_module_epoch(self): + import typing + for number, hint in enumerate(("list['Type']", "Optional['Type']", "Choice['Type']")): + with self.subTest(hint=hint): + code = ( + "from typing import Optional\nType = int\nChoice = list\n" + f"def choose(value: {hint}):\n return value\n" + "Type = str\nChoice = tuple\n" + ) + namespace = {} + exec(code, namespace) + actual = typing.get_type_hints(namespace['choose'], namespace)['value'] + if sys.version_info >= (3, 11) or number == 1: + # get_type_hints resolves strings nested in builtin + # generics since 3.11; 3.10 leaves list['Type'] as is. + self.assertIn(str, typing.get_args(actual)) + self.assertEqual(['real'], namespace['choose'](['real'])) + resource = testutils.create_module(self.project, 'nested_forward_' + str(number)) + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + if number == 1: + self.assertIsInstance(parameter.get_type(), evaluate.UnionType) + self.assertIs(builtins.get_str_type(), parameter.get_type().alternatives[0]) + else: + # `Choice` itself is read eagerly before 3.14 (list) and + # after the module ran under PEP 649 (tuple). + container = {list: builtins.List, tuple: builtins.Tuple}[typing.get_origin(actual)] + self.assertIsInstance(parameter.get_type(), container) + element = parameter.get_type().get_holding_objects()[0] + self.assertIs(builtins.get_str_type(), element.get_type()) + + def test_class_nested_forward_reference_uses_function_globals(self): + import typing + code = ( + "Type = int\nclass Owner:\n Type = bytes\n" + " def choose(self, value: list['Type']):\n return value\n" + " Type = tuple\nType = str\n" + ) + namespace = {} + exec(code, namespace) + hint = typing.get_type_hints(namespace['Owner'].choose, namespace)['value'] + # get_type_hints resolves strings nested in builtin generics since 3.11. + self.assertEqual(list[str] if sys.version_info >= (3, 11) else list['Type'], hint) + resource = testutils.create_module(self.project, 'class_nested_forward') + resource.write(code) + function = self.project.get_pymodule(resource)['Owner'].get_object()['choose'].get_object() + parameter = function.get_parameters()['value'].get_object() + self.assertIsInstance(parameter.get_type(), builtins.List) + self.assertIs(builtins.get_str_type(), parameter.get_type().holding.get_type()) + + def test_annotated_metadata_never_becomes_type_syntax(self): + import typing + for number, hint in enumerate(( + "Annotated[int, 'label] | str']", + "Annotated[int, ('metadata', 'Optional[str]')]", + '\"Annotated[int, \'label] | str\']\"', + )): + with self.subTest(hint=hint): + code = f"from typing import Annotated\ndef choose(value: {hint}):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertIs(int, typing.get_type_hints(namespace['choose'], namespace)['value']) + self.assertEqual(7, namespace['choose'](7)) + resource = testutils.create_module(self.project, 'annotated_metadata_' + str(number)) + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertIs(builtins.builtins['int'].get_object(), parameter.get_type()) + + def test_custom_class_subscription_uses_native_dispatch(self): + code = ( + "class Local:\n @classmethod\n def __class_getitem__(cls, key):\n return int\n" + "def choose(value: Local[str]):\n return value\n" + ) + namespace = {} + exec(code, namespace) + self.assertIs(int, namespace['choose'].__annotations__['value']) + resource = testutils.create_module(self.project, 'custom_class_subscription') + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertIs(builtins.builtins['int'].get_object(), parameter.get_type()) + + def test_metaclass_subscription_uses_native_dispatch(self): + code = ( + "class Meta(type):\n def __getitem__(cls, key):\n return int\n" + "class Local(metaclass=Meta): pass\n" + "def choose(value: Local[str]):\n return value\n" + ) + namespace = {} + exec(code, namespace) + self.assertIs(int, namespace['choose'].__annotations__['value']) + resource = testutils.create_module(self.project, 'metaclass_subscription') + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertIs(builtins.builtins['int'].get_object(), parameter.get_type()) + + def test_metaclass_union_overrides_do_not_become_native_unions(self): + for number, (method, expression) in enumerate((('__or__', 'Local | str'), ('__ror__', 'str | Local'))): + with self.subTest(method=method): + code = ( + f"class Parent(type):\n def {method}(cls, other):\n return int\n" + "class Meta(Parent): pass\nclass Local(metaclass=Meta): pass\n" + f"def choose(value: {expression}):\n return value\n" + ) + namespace = {} + exec(code, namespace) + self.assertIs(int, namespace['choose'].__annotations__['value']) + resource = testutils.create_module(self.project, 'custom_class_union_' + str(number)) + resource.write(code) + module = self.project.get_pymodule(resource) + parameter = module['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + self.assertIsNone(evaluate.evaluate(expression, module)) + + def test_inherited_native_union_operators_keep_metaclass_support(self): + code = ( + "class Meta(type): pass\nclass Local(metaclass=Meta): pass\n" + "def choose(value: Local | str):\n return value\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace['Local'], str), namespace['choose'].__annotations__['value'].__args__) + resource = testutils.create_module(self.project, 'native_metaclass_union') + resource.write(code) + module = self.project.get_pymodule(resource) + parameter = module['choose'].get_object().get_parameters()['value'].get_object() + self.assertIsInstance(parameter.get_type(), evaluate.UnionType) + self.assertIs(module['Local'].get_object(), parameter.get_type().alternatives[0]) + + def test_instance_alias_union_is_not_a_type_union(self): + for number, (method, expression) in enumerate((('__or__', 'Alias | str'), ('__ror__', 'str | Alias'))): + with self.subTest(method=method): + code = ( + f"class Form:\n def {method}(self, other):\n return int\n" + "Alias = Form()\n" + f"def choose(value: {expression}):\n return value\n" + ) + namespace = {} + exec(code, namespace) + self.assertIs(int, namespace['choose'].__annotations__['value']) + resource = testutils.create_module(self.project, 'instance_alias_union_' + str(number)) + resource.write(code) + module = self.project.get_pymodule(resource) + parameter = module['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + self.assertIsNone(evaluate.evaluate(expression, module)) + + def test_native_parameterized_alias_union_preserves_element_type(self): + import typing + code = "def choose(value: list[str] | None):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual((list[str], type(None)), typing.get_args(namespace['choose'].__annotations__['value'])) + self.assertEqual(['real'], namespace['choose'](['real'])) + resource = testutils.create_module(self.project, 'native_parameterized_union') + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertIsInstance(parameter.get_type(), evaluate.UnionType) + sequence = parameter.get_type().alternatives[0] + if not isinstance(sequence, pyobjects.AbstractClass): + sequence = sequence.get_type() + self.assertIsInstance(sequence, builtins.List) + self.assertIs(builtins.get_str_type(), sequence.holding.get_type()) + self.assertTrue(builtins._is_runtime_class(parameter.get_type().alternatives[1], type(None))) + + def test_named_generic_alias_materializes_native_parameter(self): + for number, declaration in enumerate(("Rows = list[str]", "Rows = list[str]\nOther = Rows\nRows = Other")): + with self.subTest(declaration=declaration): + code = declaration + "\ndef choose(value: Rows):\n return value[0].strip()\n" + namespace = {} + exec(code, namespace) + self.assertEqual('real', namespace['choose']([' real '])) + self.assertEqual((str,), namespace['choose'].__annotations__['value'].__args__) + resource = testutils.create_module(self.project, 'named_generic_alias_' + str(number)) + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + parameter = function.get_parameters()['value'].get_object() + self.assertNotIsInstance(parameter, builtins.GenericAlias) + self.assertIsInstance(parameter.get_type(), builtins.List) + self.assertIs(builtins.get_str_type(), parameter.get_type().holding.get_type()) + self.assertIs(builtins.get_str_type(), function.get_returned_object(None).get_type()) + + def test_qualified_generic_alias_materializes_native_parameter(self): + import subprocess + import sys + source = testutils.create_module(self.project, 'row_types') + source.write("Rows = list[str]\n") + resource = testutils.create_module(self.project, 'qualified_generic_alias') + resource.write("import row_types as t\ndef choose(value: t.Rows):\n return value[0].strip()\n") + execution = subprocess.run( + [sys.executable, '-c', "from qualified_generic_alias import choose; assert choose([' real ']) == 'real'; assert choose.__annotations__['value'] == list[str]"], + cwd=self.project.address, capture_output=True, text=True, + ) + self.assertEqual(0, execution.returncode, execution.stderr) + self.assertEqual('', execution.stderr) + function = self.project.get_pymodule(resource)['choose'].get_object() + parameter = function.get_parameters()['value'].get_object() + self.assertNotIsInstance(parameter, builtins.GenericAlias) + self.assertIsInstance(parameter.get_type(), builtins.List) + self.assertIs(builtins.get_str_type(), parameter.get_type().holding.get_type()) + self.assertIs(builtins.get_str_type(), function.get_returned_object(None).get_type()) + + def test_nested_generic_alias_preserves_native_holdings(self): + code = "Rows = list[tuple[str, int]]\ndef choose(value: Rows):\n return value[0][1]\n" + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['choose']([('key', 7)])) + resource = testutils.create_module(self.project, 'nested_generic_alias') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + parameter = function.get_parameters()['value'].get_object() + self.assertIsInstance(parameter.get_type(), builtins.List) + pair = parameter.get_type().holding + self.assertNotIsInstance(pair, builtins.GenericAlias) + self.assertIsInstance(pair.get_type(), builtins.Tuple) + self.assertIs(builtins.builtins['int'].get_object(), function.get_returned_object(None).get_type()) + + def test_named_generic_union_alias_preserves_none_alternative(self): + import typing + code = "Rows = list[str]\nMaybe = Rows | None\ndef choose(value: Maybe):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual((list[str], type(None)), typing.get_args(namespace['choose'].__annotations__['value'])) + self.assertIsNone(namespace['choose'](None)) + self.assertEqual(['real'], namespace['choose'](['real'])) + resource = testutils.create_module(self.project, 'named_generic_union_alias') + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertIsInstance(parameter.get_type(), evaluate.UnionType) + sequence, none_type = parameter.get_type().alternatives + self.assertIsInstance(sequence, builtins.List) + self.assertIs(builtins.get_str_type(), sequence.holding.get_type()) + self.assertTrue(builtins._is_runtime_class(none_type, type(None))) + + def test_generic_alias_with_unproven_argument_remains_unknown(self): + code = "Rows = list['Missing']\ndef choose(value: Rows):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual(['real'], namespace['choose'](['real'])) + resource = testutils.create_module(self.project, 'unproven_generic_alias') + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + self.assertNotIsInstance(parameter, builtins.GenericAlias) + + def test_mutable_type_choices_do_not_invent_selected_alias(self): + cases = ( + ("Choices = [list[str], list[int]]", "Choices[1]"), + ("Choices = [list[str]]\nChoices[0] = list[int]", "Choices[0]"), + ("Choices = [list[str]]\nChoices[0] = list[int]", "Choices.pop()"), + ("Choices = [list[str]]\nChoices[0] = list[int]", "next(iter(Choices))"), + ("Choices = []\nChoices.append(list[str])\nChoices[0] = list[int]", "Choices[0]"), + ) + for number, (declaration, selection) in enumerate(cases): + with self.subTest(selection=selection, declaration=declaration): + code = declaration + f"\nRows = {selection}\ndef choose(value: Rows):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual([7], namespace['choose']([7])) + resource = testutils.create_module(self.project, 'mutable_type_choices_' + str(number)) + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + + def test_mutable_class_choices_do_not_invent_selected_type(self): + code = "Choices = [str, int]\nSelected = Choices[1]\ndef choose(value: Selected):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertIs(int, namespace['choose'].__annotations__['value']) + self.assertEqual(7, namespace['choose'](7)) + resource = testutils.create_module(self.project, 'mutable_class_choices') + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + + def test_immutable_type_choices_keep_exact_tuple_position(self): + code = "Choices = (list[str], list[int])\nRows = Choices[1]\ndef choose(value: Rows):\n return value[0]\n" + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual(7, namespace['choose']([7])) + resource = testutils.create_module(self.project, 'immutable_type_choices') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + parameter = function.get_parameters()['value'].get_object() + self.assertIsInstance(parameter.get_type(), builtins.List) + self.assertIs(builtins.builtins['int'].get_object(), parameter.get_type().holding.get_type()) + self.assertIs(builtins.builtins['int'].get_object(), function.get_returned_object(None).get_type()) + + def test_transitive_mutable_alias_paths_remain_unknown(self): + cases = ( + ("Choices = [(list[str],), (list[int],)]", "Choices[1][0]"), + ("Choices = [(list[str],)]\nChoices[0] = (list[int],)", "Choices[0][0]"), + ("Choices = {'rows': list[str]}\nChoices['rows'] = list[int]", "Choices['rows']"), + ("Choices = ({'rows': list[str]},)\nChoices[0]['rows'] = list[int]", "Choices[0]['rows']"), + ("Choices = {'rows': (list[str],)}\nChoices['rows'] = (list[int],)", "Choices['rows'][0]"), + ("Choices = [{'rows': (list[str],)}]\nChoices[0]['rows'] = (list[int],)", "Choices[0]['rows'][0]"), + ("Choices = set(((list[str],),))\nChoices.remove((list[str],))\nChoices.add((list[int],))", "Choices.pop()[0]"), + ) + for number, (declaration, selection) in enumerate(cases): + with self.subTest(selection=selection, declaration=declaration): + code = declaration + f"\nRows = {selection}\ndef choose(value: Rows):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual([7], namespace['choose']([7])) + resource = testutils.create_module(self.project, 'transitive_mutable_alias_' + str(number)) + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + + def test_nested_immutable_alias_paths_keep_exact_positions(self): + code = "Choices = ((list[str],), (list[int],))\nRows = Choices[1][0]\ndef choose(value: Rows):\n return value[0]\n" + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual(7, namespace['choose']([7])) + resource = testutils.create_module(self.project, 'nested_immutable_alias') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertIs(builtins.builtins['int'].get_object(), function.get_returned_object(None).get_type()) + + def test_mutable_instance_alias_fields_do_not_select_first_instance(self): + classes = ( + "class Text:\n Rows = list[str]\nclass Number:\n Rows = list[int]\n", + "class Text:\n @property\n def Rows(self):\n return list[str]\nclass Number:\n @property\n def Rows(self):\n return list[int]\n", + "class TextBase:\n Rows = list[str]\nclass Text(TextBase): pass\nclass NumberBase:\n Rows = list[int]\nclass Number(NumberBase): pass\n", + "class Text:\n def __init__(self):\n self.Rows = list[str]\nclass Number:\n def __init__(self):\n self.Rows = list[int]\n", + ) + for number, declarations in enumerate(classes): + with self.subTest(number=number): + code = declarations + "Choices = [Text(), Number()]\nRows = Choices[1].Rows\ndef choose(value: Rows):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual([7], namespace['choose']([7])) + resource = testutils.create_module(self.project, 'mutable_instance_alias_' + str(number)) + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + + def test_type_alias_iterator_position_is_not_assumed(self): + cases = ( + "Choices = iter((list[str], list[int]))\nnext(Choices)\nRows = next(Choices)\n", + "def choices():\n yield list[str]\n yield list[int]\nChoices = choices()\nnext(Choices)\nRows = next(Choices)\n", + "class Text:\n Rows = list[str]\nclass Number:\n Rows = list[int]\nChoices = iter((Text(), Number()))\nnext(Choices)\nRows = next(Choices).Rows\n", + ) + for number, declarations in enumerate(cases): + with self.subTest(number=number): + code = declarations + "def choose(value: Rows):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual([7], namespace['choose']([7])) + resource = testutils.create_module(self.project, 'iterator_alias_position_' + str(number)) + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + + def test_immutable_instance_selection_preserves_exact_field(self): + code = ( + "class Text:\n Rows = list[str]\nclass Number:\n Rows = list[int]\n" + "Choices = ((Text(),), (Number(),))\nRows = Choices[1][0].Rows\n" + "def choose(value: Rows):\n return value[0]\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual(7, namespace['choose']([7])) + resource = testutils.create_module(self.project, 'immutable_instance_alias') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertIs(builtins.builtins['int'].get_object(), function.get_returned_object(None).get_type()) + + def test_sampled_receiver_methods_do_not_restore_type_certainty(self): + declarations = ( + "class Text:\n def Rows(self):\n return list[str]\n def child(self):\n return self\n" + "class Number:\n def Rows(self):\n return list[int]\n def child(self):\n return self\n" + "def identity(value):\n return value\n" + ) + cases = ( + "Choices = [Text(), Number()]\nRows = Choices[1].Rows()", + "Choices = [Text(), Number()]\nmethod = Choices[1].Rows\nRows = method()", + "Choices = [Text(), Number()]\nRows = Choices[1].child().Rows()", + "Choices = [Text(), Number()]\nRows = identity(Choices[1]).Rows()", + "Choices = [(Text(),), (Number(),)]\nRows = Choices[1][0].Rows()", + "Choices = iter((Text(), Number()))\nnext(Choices)\nRows = next(Choices).Rows()", + ) + for number, selection in enumerate(cases): + with self.subTest(number=number): + code = declarations + selection + "\ndef choose(value: Rows):\n return value\n" + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual([7], namespace['choose']([7])) + resource = testutils.create_module(self.project, 'sampled_receiver_method_' + str(number)) + resource.write(code) + parameter = self.project.get_pymodule(resource)['choose'].get_object().get_parameters()['value'].get_object() + self.assertEqual(pyobjects.get_unknown(), parameter) + + def test_exact_tuple_receiver_method_preserves_type_alias(self): + code = ( + "class Text:\n def Rows(self):\n return list[str]\n" + "class Number:\n def Rows(self):\n return list[int]\n" + "Choices = (Text(), Number())\nRows = Choices[1].Rows()\n" + "def choose(value: Rows):\n return value[0]\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual(list[int], namespace['choose'].__annotations__['value']) + self.assertEqual(7, namespace['choose']([7])) + resource = testutils.create_module(self.project, 'exact_receiver_method') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertIs(builtins.builtins['int'].get_object(), function.get_returned_object(None).get_type()) + + def test_annotated_parameters_preserve_native_string_methods(self): + code = ( + "def clean(script: str, /, *, ending: str):\n" + " return script.strip().rstrip(ending)\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual("value", namespace['clean'](' value; ', ending=';')) + resource = testutils.create_module(self.project, 'annotations') + resource.write(code) + function = self.project.get_pymodule(resource)['clean'].get_object() + for name in ('script', 'ending'): + with self.subTest(name=name): + parameter = function.get_parameters()[name].get_object() + self.assertIs(builtins.get_str_type(), parameter.get_type()) + self.assertIn('rstrip', parameter) + + def test_forward_return_annotation_uses_declared_class(self): + code = "class Label: pass\ndef choose(value) -> 'Label':\n return value\n" + namespace = {} + exec(code, namespace) + self.assertIsInstance(namespace['choose'](namespace['Label']()), namespace['Label']) + resource = testutils.create_module(self.project, 'annotations') + resource.write(code) + module = self.project.get_pymodule(resource) + returned = module['choose'].get_object().get_returned_object(None) + self.assertIs(module['Label'].get_object(), returned.get_type()) + + def test_unresolved_source_annotation_does_not_adopt_docstring_type(self): + code = ( + "def choose(value: 'Missing'):\n" + " '''\n :type value: str\n '''\n" + " return value\n" + ) + namespace = {} + exec(code, namespace) + self.assertEqual(7, namespace['choose'](7)) + resource = testutils.create_module(self.project, 'annotations') + resource.write(code) + function = self.project.get_pymodule(resource)['choose'].get_object() + self.assertEqual(pyobjects.get_unknown(), function.get_parameters()['value'].get_object()) + + class DocstringParamHintingTest(AbstractHintingTest): def test_hint_param(self): code = dedent('''\ @@ -65,6 +788,87 @@ def a_method(self, a_arg): result = self._assist(code) self.assert_completion_in_result("is_alive", "attribute", result) + def test_hint_param_on_a_hierarchy_python_refuses(self): + code = dedent('''\ + class Base(object): + def a_method(self, a_arg): + """:type a_arg: threading.Thread""" + + + class Left(Base): + pass + + + class Sample(Base, Left): + def a_method(self, a_arg): + a_arg.is_''') + result = self._assist(code) + self.assert_completion_in_result("is_alive", "attribute", result) + + def test_hint_param_follows_the_method_resolution_order(self): + code = dedent('''\ + class Base(object): + def a_method(self, a_arg): + """:type a_arg: threading.Thread""" + + + class Left(Base): + pass + + + class Right(Base): + def a_method(self, a_arg): + """:type a_arg: threading.Event""" + + + class Sample(Left, Right): + def a_method(self, a_arg): + a_arg.is_''') + result = self._assist(code) + self.assert_completion_in_result("is_set", "attribute", result) + self.assert_completion_not_in_result("is_alive", "attribute", result) + + + def test_hint_param_follows_the_method_resolution_order(self): + code = dedent('''\ + class Base(object): + def a_method(self, a_arg): + """:type a_arg: threading.Thread""" + + + class Left(Base): + pass + + + class Right(Base): + def a_method(self, a_arg): + """:type a_arg: threading.Event""" + + + class Sample(Left, Right): + def a_method(self, a_arg): + a_arg.is_''') + result = self._assist(code) + self.assert_completion_in_result("is_set", "attribute", result) + self.assert_completion_not_in_result("is_alive", "attribute", result) + + + def test_hint_param_on_a_hierarchy_python_refuses(self): + code = dedent('''\ + class Base(object): + def a_method(self, a_arg): + """:type a_arg: threading.Thread""" + + + class Left(Base): + pass + + + class Sample(Base, Left): + def a_method(self, a_arg): + a_arg.is_''') + result = self._assist(code) + self.assert_completion_in_result("is_alive", "attribute", result) class DocstringReturnHintingTest(AbstractHintingTest): def test_hierarchical_hint_rtype(self): @@ -269,14 +1073,30 @@ def test_hint_or(self): class Sample(object): {fix_indents(self._make_class_hint("str | threading.Thread"))} def a_method(self): - for i in self.a_attr.values(): - i.is_a""") + self.a_attr.is_a""") + result = self._assist(code) + self.assert_completion_in_result("is_alive", "attribute", result) + + def test_assignment_hints_ignore_names_that_are_not_assigned(self): + from rope.base import libutils + from rope.base.oi.type_hinting import utils + from rope.base.oi.type_hinting.providers.pep0484_type_comments import ( + AssignmentProvider, + ) + + pymodule = libutils.get_string_module(self.project, "def a_func():\n pass\n") + defined_name = pymodule["a_func"] + self.assertIsNone(AssignmentProvider(None)(defined_name)) + self.assertIsNone(utils.get_class_with_attr_name(defined_name)) + + def test_hint_or_keeps_first_alternative_attributes(self): + code = dedent(f"""\ + class Sample(object): + {fix_indents(self._make_class_hint("str | threading.Thread"))} + def a_method(self): + self.a_attr.upp""") result = self._assist(code) - try: - # Be sure, there isn't errors currently - self.assert_completion_in_result('is_alive', 'attribute', result) - except AssertionError as e: - pytest.xfail("failing configuration (but should work)") + self.assert_completion_in_result("upper", "builtin", result) def test_hint_nonexistent(self): code = dedent(f"""\ diff --git a/ropetest/value_descriptor_test.py b/ropetest/value_descriptor_test.py new file mode 100644 index 000000000..9aeb8477e --- /dev/null +++ b/ropetest/value_descriptor_test.py @@ -0,0 +1,105 @@ +"""Source inference agrees with Python's unchanged-value descriptors.""" + +from textwrap import dedent + +import pytest + +from rope.base import libutils +from rope.base.project import Project + +SOURCE = dedent("""\ + def native(value): + return value + class Export: + def __init__(self, value): + self._value = value + def __get__(self, instance, owner=None): + return self._value + class Owner: + exported = Export({argument}) + class Derived(Owner): + pass + selected = {access}.exported +""") + + +@pytest.mark.parametrize("access", ["Owner", "Owner()", "Derived", "Derived()"]) +@pytest.mark.parametrize("argument", ["native", "value=native"]) +def test_native_value_descriptor_preserves_identity(tmp_path, access, argument): + source = SOURCE.format(access=access, argument=argument) + namespace = {} + exec(source, namespace) + assert namespace["selected"] is namespace["native"] + project = Project(str(tmp_path), ropefolder=None) + try: + scope = libutils.get_string_scope(project, source) + assert scope["selected"].get_object() is scope["native"].get_object() + finally: + project.close() + + +@pytest.mark.parametrize("access", ["Owner", "Owner()"]) +@pytest.mark.parametrize("transformation", ["getter", "initializer"]) +def test_transforming_descriptor_does_not_reuse_constructor_identity( + tmp_path, access, transformation +): + source = SOURCE.format(access=access, argument="native") + if transformation == "getter": + source = source.replace("return self._value", "return lambda value: value") + else: + source = source.replace("self._value = value", "self._value = lambda value: value") + namespace = {} + exec(source, namespace) + assert namespace["selected"] is not namespace["native"] + project = Project(str(tmp_path), ropefolder=None) + try: + scope = libutils.get_string_scope(project, source) + assert scope["selected"].get_object() is not scope["native"].get_object() + finally: + project.close() + + +@pytest.mark.parametrize("access", ["Owner", "Owner()"]) +@pytest.mark.parametrize( + "rebinding", + [ + " __get__ = lambda self, instance, owner=None: str\n", + " __init__ = lambda self, value: setattr(self, '_value', str)\n", + ], +) +def test_rebound_descriptor_method_does_not_reuse_constructor_identity( + tmp_path, access, rebinding +): + source = SOURCE.format(access=access, argument="native").replace( + " return self._value\n", + " return self._value\n" + rebinding, + ) + namespace = {} + exec(source, namespace) + assert namespace["selected"] is not namespace["native"] + project = Project(str(tmp_path), ropefolder=None) + try: + scope = libutils.get_string_scope(project, source) + assert scope["selected"].get_object() is not scope["native"].get_object() + finally: + project.close() + + +def test_attribute_write_index_tracks_recaptured_native_source(tmp_path): + project = Project(str(tmp_path), ropefolder=None) + try: + resource = project.root.create_file("subject.py") + source = SOURCE.format(access="Owner", argument="native") + resource.write(source) + old = project.get_pymodule(resource).get_source_binding_index() + assert old.attribute_writes("exported") == () + updated = source.replace("selected =", "Owner.exported = native\nselected =") + namespace = {} + exec(updated, namespace) + assert namespace["selected"] is namespace["native"] + resource.write(updated) + new = project.get_pymodule(resource).get_source_binding_index() + assert len(new.attribute_writes("exported")) == 1 + assert old.attribute_writes("exported") == () + finally: + project.close()