From c4757afd6c1496df9221d2704968677220332fbf Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Fri, 6 Nov 2026 02:12:39 +0000 Subject: [PATCH] gh-129724: Intern code object names before merging them with constants makecode() merges the co_names and co_localsplusnames tuples with the constant cache, so they can become an equal tuple constant of another code object, and _PyCode_New() then interns the items of that tuple in place. If an equal string was already interned in the process, the string of the constant was replaced by it; otherwise the string of the constant itself became the interned one. The constants which are the same object, and so the marshal output, depended on the process state. The parser interns identifiers and the compiler interns the ASCII identifier-like string constants, so this needs a name which is not interned yet and is not an ASCII identifier: a private name of a class with a non-ASCII name, which each compilation mangles into a new string, or an identifier of an AST passed to compile() which was not created by the parser. Intern the names before the merge. --- Lib/test/test_compile.py | 46 +++++++++++++++++++ ...-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst | 4 ++ Python/assemble.c | 26 +++++++++++ 3 files changed, 76 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index df766387dd4c72..d2069a0ac4214d 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2342,6 +2342,29 @@ def g(self, __x): return [__x] """, strings=['_ID_A__x']), + # A constant of an earlier method is equal to the co_names of a + # later one. The mangled name is not an ASCII identifier, so the + # compiler does not intern the equal string constants. + 'mangled_names_non_ascii': Case( + source=""" + class ID_\xe9: + def g(self): + return ("_ID_\xe9__x",), "_ID_\xe9__x" + def f(self): + return self.__x + """, + strings=['_ID_\xe9__x']), + # The same with the names of local variables. + 'mangled_varnames_non_ascii': Case( + source=""" + class ID_\xe9: + def g(self): + return ("_ID_\xe9__x",), "_ID_\xe9__x" + @staticmethod + def f(__x): + return __x + """, + strings=['_ID_\xe9__x']), # The names of the keyword arguments are a constant equal to # the co_names of the function. 'kwnames': Case( @@ -2548,6 +2571,29 @@ def code_constants(code): self.assertEqual(len(names), 1) self.assertTrue(any(const is names for const in consts)) + def test_ast_names_not_interned(self): + # The identifiers of an AST which was not created by the parser are + # not interned yet. A constant of an earlier function is equal to + # the co_names of a later one. + source = textwrap.dedent(""" + def g(self): + return ("gh129724_ast_\xe9",), "gh129724_ast_\xe9" + def f(self): + return self.placeholder + """) + keep = [] + dumps = [] + for attempt in range(2): + tree = ast.parse(source) + attribute, = [node for node in ast.walk(tree) + if isinstance(node, ast.Attribute)] + # Not a literal: a new string object, which is not interned. + attribute.attr = ''.join(['gh129724_ast_', '\xe9']) + code = compile(tree, '', 'exec') + keep.append(code) + dumps.append(marshal.dumps(code)) + self.assertEqual(dumps[0], dumps[1]) + @support.cpython_only def test_slice_strings(self): # The strings of a slice constant are interned like the other diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst new file mode 100644 index 00000000000000..a259af1dcea0f9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst @@ -0,0 +1,4 @@ +Fix the :mod:`marshal` output of compiled code, and so ``.pyc`` files, +differing for the same source depending on what ran before in the process. +This affected private names in classes with non-ASCII names, and +:func:`compile` of an AST with names not created by :func:`ast.parse`. diff --git a/Python/assemble.c b/Python/assemble.c index 8f68e233ab7b93..9f4ffb4050d259 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -4,7 +4,10 @@ #include "pycore_instruction_sequence.h" #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE #include "pycore_opcode_metadata.h" // is_pseudo_target, _PyOpcode_Caches +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_symtable.h" // _Py_SourceLocation +#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() #include @@ -503,6 +506,27 @@ dict_keys_inorder(PyObject *dict, Py_ssize_t offset) extern void _Py_set_localsplus_info(int, PyObject *, unsigned char, PyObject *, PyObject *); +/* Intern the names in *tuple*, as _PyCode_New() does, before the tuple is + * merged with the constant cache. + * + * The merge can replace the tuple by an equal tuple constant of another + * code object, whose items _PyCode_New() then interns in place. A string + * of that constant would become the interned string or be replaced by it, + * depending on whether an equal string was already interned in the process, + * and so would the output of marshal for the same source. Names which the + * parser did not create, such as mangled private names or the identifiers + * of an AST passed to compile(), may not be interned yet. + */ +static void +intern_names(PyObject *tuple) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(tuple); i++) { + assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(tuple, i))); + _PyUnicode_InternImmortal(interp, &_PyTuple_ITEMS(tuple)[i]); + } +} + static int compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, int flags, PyObject *names, PyObject *kinds) @@ -610,6 +634,7 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ if (!names) { goto error; } + intern_names(names); if (_PyCompile_ConstCacheMergeOne(const_cache, &names) < 0) { goto error; } @@ -674,6 +699,7 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ goto error; } + intern_names(localsplusnames); if (_PyCompile_ConstCacheMergeOne(const_cache, &localsplusnames) < 0) { goto error; }