diff --git a/markdown_it/rules_core/normalize.py b/markdown_it/rules_core/normalize.py index 32439243..e9fb0273 100644 --- a/markdown_it/rules_core/normalize.py +++ b/markdown_it/rules_core/normalize.py @@ -5,15 +5,18 @@ from .state_core import StateCore # https://spec.commonmark.org/0.29/#line-ending -NEWLINES_RE = re.compile(r"\r\n?|\n") -NULL_RE = re.compile(r"\0") +NEWLINES_RE = re.compile(r"\r\n?") def normalize(state: StateCore) -> None: - # Normalize newlines - string = NEWLINES_RE.sub("\n", state.src) + string = state.src + + # Normalize newlines. Most input has no "\r", so skip the substitution then. + if "\r" in string: + string = NEWLINES_RE.sub("\n", string) # Replace NULL characters - string = NULL_RE.sub("\ufffd", string) + if "\0" in string: + string = string.replace("\0", "\ufffd") state.src = string diff --git a/tests/test_api/test_main.py b/tests/test_api/test_main.py index 74c7b0f8..560aa154 100644 --- a/tests/test_api/test_main.py +++ b/tests/test_api/test_main.py @@ -633,3 +633,24 @@ def test_src_char_code_follows_src(): state.src = "xyz" with pytest.warns(DeprecationWarning): assert list(state.srcCharCode) == [120, 121, 122] + + +@pytest.mark.parametrize( + "src,expected", + [ + ("a\r\nb", "a\nb"), + ("a\rb", "a\nb"), + ("a\r\r\nb", "a\n\nb"), + ("a\n\rb", "a\n\nb"), + ("a\0b\r\n\0", "a�b\n�"), + ("plain\ntext", "plain\ntext"), + ], +) +def test_normalize(src, expected): + """Line endings become ``\\n`` and NULL characters become U+FFFD.""" + from markdown_it.rules_core.normalize import normalize + from markdown_it.rules_core.state_core import StateCore + + state = StateCore(src, MarkdownIt(), {}) + normalize(state) + assert state.src == expected