From 2e566ef484c6580e67e500eab5dbd718c7af0848 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Sat, 21 Nov 2026 23:37:31 -0300 Subject: [PATCH 1/3] Start autoimport cache workers without forking a threaded process `AutoImport.generate_cache` and `generate_modules_cache` used the platform's default process start method, which is `fork` on POSIX before Python 3.14. Forking a process that runs other threads, as editors, language servers and test runners do, can deadlock the child, and Python 3.12+ warns on every such fork. Start the pool's workers with `forkserver`, or `spawn` where it is unavailable. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + rope/contrib/autoimport/sqlite.py | 17 +++++++- ropetest/contrib/autoimport/autoimporttest.py | 40 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88e4748e2..36a0ffa0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - #937 Preserve non-ASCII argument expressions and default values when changing function signatures (@Jinyong-Li) - #826, #940 Keep star imports in other modules when inlining a function or method (@chinghar) - #948, #949 Don't treat string prefixes such as the `f` in `f"..."` as occurrences when renaming a variable named `f`, `b`, `u`, `rf`, etc. (@aliassar) +- #TBD Start autoimport cache workers with `forkserver` (or `spawn`) instead of forking a possibly threaded process (@marlon-costa-dc) # Release 1.15.1 diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index 7ae8fb718..ded958213 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -4,6 +4,7 @@ import contextlib import json +import multiprocessing import re import secrets import sqlite3 @@ -55,11 +56,23 @@ from collections.abc import Collection +def _process_pool() -> ProcessPoolExecutor: + """Return a pool whose workers never come from forking this process. + + Forking a process that has threads (an editor, a language server or a + test runner usually do) can deadlock the child. ``forkserver`` (the + POSIX default since Python 3.14) or ``spawn`` start workers cleanly. + """ + methods = multiprocessing.get_all_start_methods() + method = "forkserver" if "forkserver" in methods else "spawn" + return ProcessPoolExecutor(mp_context=multiprocessing.get_context(method)) + + def get_future_names( packages: List[Package], underlined: bool, job_set: taskhandle.BaseJobSet ) -> Generator[Future[Collection[Name]], None, None]: """Get all names as futures.""" - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for package in packages: for module in get_files(package, underlined): job_set.started_job(module.modname) @@ -390,7 +403,7 @@ def generate_cache( models.Package.delete_by_package_name, (self.project_package.name,) ) futures = [] - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for file in resources: job_set.started_job(f"Working on {file.path}") module = self._resource_to_module(file, underlined) diff --git a/ropetest/contrib/autoimport/autoimporttest.py b/ropetest/contrib/autoimport/autoimporttest.py index 072ab1ed8..64f66dc5c 100644 --- a/ropetest/contrib/autoimport/autoimporttest.py +++ b/ropetest/contrib/autoimport/autoimporttest.py @@ -1,4 +1,7 @@ +import multiprocessing import sqlite3 +import threading +import warnings from concurrent.futures import ThreadPoolExecutor from contextlib import closing, contextmanager from textwrap import dedent @@ -101,6 +104,43 @@ def foo(): assert [("from pkg1 import foo", "foo")] == results +@pytest.fixture +def fork_default_start_method(): + # Select "fork", the POSIX default before Python 3.14, so the test shows + # the cache workers do not depend on the global start method. + if "fork" not in multiprocessing.get_all_start_methods(): + pytest.skip("the platform has no fork start method") + previous = multiprocessing.get_start_method(allow_none=True) + multiprocessing.set_start_method("fork", force=True) + yield + multiprocessing.set_start_method(previous, force=True) + + +@pytest.mark.usefixtures("fork_default_start_method") +def test_cache_workers_do_not_fork_a_threaded_process( + autoimport: AutoImport, + pkg1: Folder, +): + # Forking a process that runs other threads can deadlock the child. + # CPython reports that warning from inside fork(), where it cannot be + # raised, so the test records warnings instead of relying on the + # "error" filter. + pkg1.get_child("__init__.py").write("def foo():\n pass\n") + stop = threading.Event() + helper = threading.Thread(target=stop.wait) + helper.start() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + autoimport.generate_cache([pkg1.get_child("__init__.py")]) + autoimport.generate_modules_cache(["textwrap"]) + finally: + stop.set() + helper.join() + assert [str(warning.message) for warning in caught] == [] + assert ("from pkg1 import foo", "foo") in autoimport.search("foo", True) + + def test_multithreading( autoimport: AutoImport, project: Project, From f1d398a28cf3c31be126ee168897fbde902bb521 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Sat, 21 Nov 2026 23:38:13 -0300 Subject: [PATCH 2/3] Use this PR's number in the changelog Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a0ffa0c..1e4a69886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - #937 Preserve non-ASCII argument expressions and default values when changing function signatures (@Jinyong-Li) - #826, #940 Keep star imports in other modules when inlining a function or method (@chinghar) - #948, #949 Don't treat string prefixes such as the `f` in `f"..."` as occurrences when renaming a variable named `f`, `b`, `u`, `rf`, etc. (@aliassar) -- #TBD Start autoimport cache workers with `forkserver` (or `spawn`) instead of forking a possibly threaded process (@marlon-costa-dc) +- #964 Start autoimport cache workers with `forkserver` (or `spawn`) instead of forking a possibly threaded process (@marlon-costa-dc) # Release 1.15.1 From 4f7ea41e96b834e5c49ce9892aa4458190efcd30 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Sat, 21 Nov 2026 23:44:25 -0300 Subject: [PATCH 3/3] Assert only the fork-with-threads warning in the start-method test On macOS the recorded warnings also held `ResourceWarning: unclosed database` for sqlite connections other tests leave to the garbage collector. The test is about forking a threaded process, so assert that no "use of fork()" warning appears. Co-Authored-By: Claude Opus 5.5 --- ropetest/contrib/autoimport/autoimporttest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ropetest/contrib/autoimport/autoimporttest.py b/ropetest/contrib/autoimport/autoimporttest.py index 64f66dc5c..91cf3a9b2 100644 --- a/ropetest/contrib/autoimport/autoimporttest.py +++ b/ropetest/contrib/autoimport/autoimporttest.py @@ -137,7 +137,12 @@ def test_cache_workers_do_not_fork_a_threaded_process( finally: stop.set() helper.join() - assert [str(warning.message) for warning in caught] == [] + fork_warnings = [ + str(warning.message) + for warning in caught + if "use of fork()" in str(warning.message) + ] + assert fork_warnings == [] assert ("from pkg1 import foo", "foo") in autoimport.search("foo", True)