From bb4bbce2dc3d7baaf09cce395c94a39eb36a5487 Mon Sep 17 00:00:00 2001 From: g4lb Date: Tue, 10 Nov 2026 07:47:36 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8C=20Skip=20the=20substitutions=20in?= =?UTF-8?q?=20normalize=20when=20there=20is=20nothing=20to=20replace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- markdown_it/rules_core/normalize.py | 13 ++++++++----- tests/test_api/test_main.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/markdown_it/rules_core/normalize.py b/markdown_it/rules_core/normalize.py index 32439243..e9fb0273 100644 --- a/markdown_it/rules_core/normalize.py +++ b/markdown_it/rules_core/normalize.py @@ -5,15 +5,18 @@ from .state_core import StateCore # https://spec.commonmark.org/0.29/#line-ending -NEWLINES_RE = re.compile(r"\r\n?|\n") -NULL_RE = re.compile(r"\0") +NEWLINES_RE = re.compile(r"\r\n?") def normalize(state: StateCore) -> None: - # Normalize newlines - string = NEWLINES_RE.sub("\n", state.src) + string = state.src + + # Normalize newlines. Most input has no "\r", so skip the substitution then. + if "\r" in string: + string = NEWLINES_RE.sub("\n", string) # Replace NULL characters - string = NULL_RE.sub("\ufffd", string) + if "\0" in string: + string = string.replace("\0", "\ufffd") state.src = string diff --git a/tests/test_api/test_main.py b/tests/test_api/test_main.py index 74c7b0f8..560aa154 100644 --- a/tests/test_api/test_main.py +++ b/tests/test_api/test_main.py @@ -633,3 +633,24 @@ def test_src_char_code_follows_src(): state.src = "xyz" with pytest.warns(DeprecationWarning): assert list(state.srcCharCode) == [120, 121, 122] + + +@pytest.mark.parametrize( + "src,expected", + [ + ("a\r\nb", "a\nb"), + ("a\rb", "a\nb"), + ("a\r\r\nb", "a\n\nb"), + ("a\n\rb", "a\n\nb"), + ("a\0b\r\n\0", "a�b\n�"), + ("plain\ntext", "plain\ntext"), + ], +) +def test_normalize(src, expected): + """Line endings become ``\\n`` and NULL characters become U+FFFD.""" + from markdown_it.rules_core.normalize import normalize + from markdown_it.rules_core.state_core import StateCore + + state = StateCore(src, MarkdownIt(), {}) + normalize(state) + assert state.src == expected