diff --git a/CHANGELOG.md b/CHANGELOG.md index 88e4748e2..1e4a69886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - #937 Preserve non-ASCII argument expressions and default values when changing function signatures (@Jinyong-Li) - #826, #940 Keep star imports in other modules when inlining a function or method (@chinghar) - #948, #949 Don't treat string prefixes such as the `f` in `f"..."` as occurrences when renaming a variable named `f`, `b`, `u`, `rf`, etc. (@aliassar) +- #964 Start autoimport cache workers with `forkserver` (or `spawn`) instead of forking a possibly threaded process (@marlon-costa-dc) # Release 1.15.1 diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index 7ae8fb718..ded958213 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -4,6 +4,7 @@ import contextlib import json +import multiprocessing import re import secrets import sqlite3 @@ -55,11 +56,23 @@ from collections.abc import Collection +def _process_pool() -> ProcessPoolExecutor: + """Return a pool whose workers never come from forking this process. + + Forking a process that has threads (an editor, a language server or a + test runner usually do) can deadlock the child. ``forkserver`` (the + POSIX default since Python 3.14) or ``spawn`` start workers cleanly. + """ + methods = multiprocessing.get_all_start_methods() + method = "forkserver" if "forkserver" in methods else "spawn" + return ProcessPoolExecutor(mp_context=multiprocessing.get_context(method)) + + def get_future_names( packages: List[Package], underlined: bool, job_set: taskhandle.BaseJobSet ) -> Generator[Future[Collection[Name]], None, None]: """Get all names as futures.""" - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for package in packages: for module in get_files(package, underlined): job_set.started_job(module.modname) @@ -390,7 +403,7 @@ def generate_cache( models.Package.delete_by_package_name, (self.project_package.name,) ) futures = [] - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for file in resources: job_set.started_job(f"Working on {file.path}") module = self._resource_to_module(file, underlined) diff --git a/ropetest/contrib/autoimport/autoimporttest.py b/ropetest/contrib/autoimport/autoimporttest.py index 072ab1ed8..91cf3a9b2 100644 --- a/ropetest/contrib/autoimport/autoimporttest.py +++ b/ropetest/contrib/autoimport/autoimporttest.py @@ -1,4 +1,7 @@ +import multiprocessing import sqlite3 +import threading +import warnings from concurrent.futures import ThreadPoolExecutor from contextlib import closing, contextmanager from textwrap import dedent @@ -101,6 +104,48 @@ def foo(): assert [("from pkg1 import foo", "foo")] == results +@pytest.fixture +def fork_default_start_method(): + # Select "fork", the POSIX default before Python 3.14, so the test shows + # the cache workers do not depend on the global start method. + if "fork" not in multiprocessing.get_all_start_methods(): + pytest.skip("the platform has no fork start method") + previous = multiprocessing.get_start_method(allow_none=True) + multiprocessing.set_start_method("fork", force=True) + yield + multiprocessing.set_start_method(previous, force=True) + + +@pytest.mark.usefixtures("fork_default_start_method") +def test_cache_workers_do_not_fork_a_threaded_process( + autoimport: AutoImport, + pkg1: Folder, +): + # Forking a process that runs other threads can deadlock the child. + # CPython reports that warning from inside fork(), where it cannot be + # raised, so the test records warnings instead of relying on the + # "error" filter. + pkg1.get_child("__init__.py").write("def foo():\n pass\n") + stop = threading.Event() + helper = threading.Thread(target=stop.wait) + helper.start() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + autoimport.generate_cache([pkg1.get_child("__init__.py")]) + autoimport.generate_modules_cache(["textwrap"]) + finally: + stop.set() + helper.join() + fork_warnings = [ + str(warning.message) + for warning in caught + if "use of fork()" in str(warning.message) + ] + assert fork_warnings == [] + assert ("from pkg1 import foo", "foo") in autoimport.search("foo", True) + + def test_multithreading( autoimport: AutoImport, project: Project,