From f4dfcd5202409fe908c3bca0230cbcf1fb23639b Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:53:05 -0500 Subject: [PATCH 1/2] fix(crowdin): preserve whitespace within translation pipeline --- README.md | 12 +++ src/js/crowdin.js | 234 ++++++++++-------------------------------- tests/crowdin.test.js | 189 +++++++++++++++++++--------------- 3 files changed, 173 insertions(+), 262 deletions(-) diff --git a/README.md b/README.md index 0f1d4da..a293d4e 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,18 @@ icon filenames that contain spaces or other reserved characters. npm install @lizardbyte/shared-web --ignore-scripts ``` +## CrowdIn whitespace preservation + +The shared initializer preserves the source text's leading and trailing whitespace +inside the translation pipeline, including missing-key fallbacks and instant +translations. This covers multiline text around links, emphasis, and inline code. +It does not install a whitespace repair observer: the renderer caches and displays +the same result, avoiding the observer loop that could freeze documentation pages. + +Crowdin still loads after the page finishes loading. Preformatted code and Doxygen +`.fragment` blocks remain excluded from translation so syntax highlighters can +update them without losing their formatting. + ## CrowdIn in JSDoc Create an initializer such as `docs/crowdin-init.js`: diff --git a/src/js/crowdin.js b/src/js/crowdin.js index 65d014e..2ab724c 100644 --- a/src/js/crowdin.js +++ b/src/js/crowdin.js @@ -12,198 +12,71 @@ const CROWDIN_DIST_MIRROR = 'https://cdn.jsdelivr.net/gh/LizardByte/i18n@dist'; const CROWDIN_CODE_BLOCK_CLASS = 'crowdin-ignore-code'; const CROWDIN_PLATFORM_STYLING_MAX_ATTEMPTS = 100; const CROWDIN_PLATFORM_STYLING_RETRY_DELAY_MS = 50; -const CROWDIN_INLINE_ELEMENT_SELECTOR = [ - 'a', - 'abbr', - 'b', - 'cite', - 'code', - 'del', - 'em', - 'i', - 'ins', - 'kbd', - 'mark', - 'q', - 's', - 'samp', - 'small', - 'span', - 'strong', - 'sub', - 'sup', - 'time', - 'u', - 'var', -].join(','); +const CROWDIN_WHITESPACE_POST_PROCESSOR = 'crowdin-preserve-whitespace'; +const CROWDIN_WHITESPACE_RENDERERS = new WeakSet(); /** - * Returns a sibling when it is an inline element whose boundary Crowdin may rewrite. - * @param {Node|null} sibling Candidate sibling. - * @returns {Element|null} Matching inline element. + * Preserves source whitespace in translation results before i18nextify records + * or renders them. Its cleanWhitespace handling only recognizes single-line + * text; multiline missing keys and translations can lose their boundary spaces. + * @param {*} value Translated value. + * @param {string|string[]} key Translation lookup keys. + * @param {Object} options Translation options, including the source defaultValue. + * @returns {*} Translation with the source's exact boundary whitespace. */ -function _getCrowdinInlineSibling(sibling) { - if (!(sibling instanceof globalThis.Element)) return null; - return sibling.matches(CROWDIN_INLINE_ELEMENT_SELECTOR) ? sibling : null; +function _preserveCrowdinWhitespace(value, key, options) { + const source = options?.defaultValue; + if (typeof value !== 'string' || typeof source !== 'string') return value; + if (source.trim() === '') return source; + + const leadingWhitespace = source.slice(0, source.length - source.trimStart().length); + const trailingWhitespace = source.slice(source.trimEnd().length); + return leadingWhitespace + value.trim() + trailingWhitespace; } /** - * Returns the exact whitespace at the start of a string. - * @param {string} text Text to inspect. - * @returns {string} Leading whitespace. + * Adds a postprocessor to Crowdin's renderer before its initial translation. + * Keeping i18nextify's cached translation and DOM text identical avoids the + * mutation-observer loop caused by repairing whitespace after rendering. */ -function _getLeadingWhitespace(text) { - const trimmedText = text.trimStart(); - return text.slice(0, text.length - trimmedText.length); -} - -/** - * Returns the exact whitespace at the end of a string. - * @param {string} text Text to inspect. - * @returns {string} Trailing whitespace. - */ -function _getTrailingWhitespace(text) { - const trimmedText = text.trimEnd(); - return text.slice(trimmedText.length); -} - -/** - * Records whitespace that separates text from inline elements before Crowdin translates the page. - * @returns {Array<{ - * node: Text, - * leading: boolean, - * leadingWhitespace: string, - * trailing: boolean, - * trailingWhitespace: string, - * previousInline: Element|null, - * nextInline: Element|null, - * whitespaceOnly: boolean, - * lastRestoredNode: Text|null, - * lastRestoredFrom: string|null - * }>} Recorded text-node boundaries. - */ -function _captureCrowdinWhitespaceBoundaries() { - const boundaries = []; - const walker = document.createTreeWalker(document.body, globalThis.NodeFilter.SHOW_TEXT); - let node = walker.nextNode(); - - while (node !== null) { - const previousInline = _getCrowdinInlineSibling(node.previousSibling); - const nextInline = _getCrowdinInlineSibling(node.nextSibling); - const leadingWhitespace = previousInline === null ? '' : _getLeadingWhitespace(node.data); - const trailingWhitespace = nextInline === null ? '' : _getTrailingWhitespace(node.data); - const leading = leadingWhitespace !== ''; - const trailing = trailingWhitespace !== ''; - - if (leading || trailing) { - boundaries.push({ - node, - leading, - leadingWhitespace, - trailing, - trailingWhitespace, - previousInline, - nextInline, - whitespaceOnly: node.data.trim() === '', - lastRestoredNode: null, - lastRestoredFrom: null, - }); - } - - node = walker.nextNode(); - } - - return boundaries; -} - -/** - * Finds the current text node for a boundary after Crowdin changes the DOM. - * @param {Object} boundary Recorded whitespace boundary. - * @returns {Text|null} Current or recreated text node. - */ -function _resolveCrowdinWhitespaceNode(boundary) { - if (boundary.node.isConnected) return boundary.node; - - const previousReplacement = boundary.previousInline?.nextSibling; - if (previousReplacement instanceof globalThis.Text && previousReplacement.isConnected) { - boundary.node = previousReplacement; - return boundary.node; - } - const nextReplacement = boundary.nextInline?.previousSibling; - if (nextReplacement instanceof globalThis.Text && nextReplacement.isConnected) { - boundary.node = nextReplacement; - return boundary.node; - } - if (!boundary.whitespaceOnly) return null; - - const replacement = document.createTextNode(''); - if (boundary.previousInline?.isConnected) { - boundary.previousInline.after(replacement); - } else if (boundary.nextInline?.isConnected) { - boundary.nextInline.before(replacement); - } else { - return null; - } - - boundary.node = replacement; - return boundary.node; -} - -/** - * Restores whitespace that Crowdin removed from translated text-node boundaries. - * @param {Array} boundaries Recorded text-node boundaries. - */ -function _restoreCrowdinWhitespaceBoundaries(boundaries) { - boundaries.forEach((boundary) => { - const node = _resolveCrowdinWhitespaceNode(boundary); - if (node === null) return; - if (boundary.lastRestoredNode === node && boundary.lastRestoredFrom === node.data) return; - - const originalText = node.data; - - const currentLeadingWhitespace = _getLeadingWhitespace(node.data); - if (boundary.leading && currentLeadingWhitespace !== boundary.leadingWhitespace) { - node.data = boundary.leadingWhitespace + node.data.slice(currentLeadingWhitespace.length); - } - const currentTrailingWhitespace = _getTrailingWhitespace(node.data); - if (boundary.trailing && currentTrailingWhitespace !== boundary.trailingWhitespace) { - const translatedTextEnd = node.data.length - currentTrailingWhitespace.length; - node.data = node.data.slice(0, translatedTextEnd) + boundary.trailingWhitespace; - } - if (node.data !== originalText) { - // A translator may observe this correction and remove the same - // whitespace again. Do not repeat the correction indefinitely. - boundary.lastRestoredNode = node; - boundary.lastRestoredFrom = originalText; - } +function _installCrowdinWhitespacePreserver() { + const renderer = globalThis.i18nextify; + if (CROWDIN_WHITESPACE_RENDERERS.has(renderer)) return; + + renderer.i18next.use({ + type: 'postProcessor', + name: CROWDIN_WHITESPACE_POST_PROCESSOR, + process: _preserveCrowdinWhitespace, }); -} - -/** - * Creates the Website Translator callback and observes later translation mutations. - * @param {Array} boundaries Recorded text-node boundaries. - * @returns {Function} Website Translator callback. - */ -function _createCrowdinTranslationCallback(boundaries) { - const translationObserver = new globalThis.MutationObserver(restoreAndObserve); - function restoreAndObserve() { - translationObserver.disconnect(); - _restoreCrowdinWhitespaceBoundaries(boundaries); - translationObserver.observe(document.body, { - characterData: true, - childList: true, - subtree: true, - }); - } - - return restoreAndObserve; + const originalInit = renderer.init; + renderer.init = function(options, ...args) { + const postProcess = options.postProcess ?? []; + const rendererOptions = { + ...options, + postProcess: [ + ...(Array.isArray(postProcess) ? postProcess : [postProcess]), + CROWDIN_WHITESPACE_POST_PROCESSOR, + ], + }; + // i18next calls this handler after postprocessing a missing key, so + // preserve the boundaries of fallback and instant translations too. + if (typeof options.parseMissingKeyHandler === 'function') { + const originalHandler = options.parseMissingKeyHandler; + rendererOptions.parseMissingKeyHandler = function(key, defaultValue, translationOptions) { + const value = originalHandler.call(this, key, defaultValue, translationOptions); + return _preserveCrowdinWhitespace(value, key, translationOptions); + }; + } + return originalInit.call(this, rendererOptions, ...args); + }; + CROWDIN_WHITESPACE_RENDERERS.add(renderer); } /** * Keeps preformatted examples out of Crowdin's text-node rewriting. Syntax - * highlighters can replace their children after translation starts, leaving - * the whitespace boundary snapshots pointing at detached nodes. + * highlighters can replace their children after translation starts. Leave their + * markup and whitespace under the highlighter's control. */ function _excludeCrowdinCodeBlocks() { document.querySelectorAll('pre, .fragment').forEach((block) => { @@ -356,11 +229,10 @@ function initCrowdIn(project = 'LizardByte', platform = 'jekyll') { // Initialize Crowdin translator _excludeCrowdinCodeBlocks(); - const whitespaceBoundaries = _captureCrowdinWhitespaceBoundaries(); + _installCrowdinWhitespacePreserver(); globalThis.proxyTranslator.init({ baseUrl: currentBaseUrl, - callback: _createCrowdinTranslationCallback(whitespaceBoundaries), crawlerIgnoreClasses: [CROWDIN_CODE_BLOCK_CLASS], distribution: projectSettings[project].distribution, defaultLanguage: "en", diff --git a/tests/crowdin.test.js b/tests/crowdin.test.js index c87f1b2..ba03065 100644 --- a/tests/crowdin.test.js +++ b/tests/crowdin.test.js @@ -46,6 +46,11 @@ describe('initCrowdIn', () => { init: jest.fn() }; + globalThis.i18nextify = { + i18next: { use: jest.fn() }, + init: jest.fn(), + }; + // Use fake timers to control setTimeout jest.useFakeTimers(); @@ -128,7 +133,6 @@ describe('initCrowdIn', () => { const options = globalThis.proxyTranslator.init.mock.calls[0][0]; expect(Object.keys(options).sort()).toEqual([ 'baseUrl', - 'callback', 'crawlerIgnoreClasses', 'defaultLanguage', 'distribution', @@ -172,113 +176,131 @@ describe('initCrowdIn', () => { ]); }); - it('should restore whitespace around translated inline elements', () => { - globalThis.document.body.innerHTML = ` -

- GitHub Discussions are available. Yearly: $14.99, billed. -

-

Before removed

-

gone kept

-

gone also gone

- `; + it('should install whitespace preservation before Crowdin initializes', () => { + globalThis.proxyTranslator.init.mockImplementation(() => { + expect(globalThis.i18nextify.i18next.use).toHaveBeenCalledWith(expect.objectContaining({ + type: 'postProcessor', + name: 'crowdin-preserve-whitespace', + process: expect.any(Function), + })); + }); initCrowdIn(); jest.runAllTimers(); - - const options = globalThis.proxyTranslator.init.mock.calls[0][0]; - const translated = globalThis.document.getElementById('translated'); - const emphasis = translated.querySelector('em'); - const strongElements = translated.querySelectorAll('strong'); - const detachedBoundary = globalThis.document.querySelector('#detached i').previousSibling; - const nextAnchor = globalThis.document.getElementById('next-anchor'); - const nextAnchorWhitespace = nextAnchor.querySelector('i').nextSibling; - const noAnchor = globalThis.document.getElementById('no-anchor'); - - emphasis.previousSibling.replaceWith(document.createTextNode('GitHub')); - emphasis.nextSibling.replaceWith(document.createTextNode('are available. ')); - strongElements[0].nextSibling.remove(); - detachedBoundary.remove(); - nextAnchor.querySelector('i').remove(); - nextAnchorWhitespace.remove(); - noAnchor.remove(); - - options.callback(); - options.callback(); - - expect(translated.textContent.trim()).toBe( - 'GitHub Discussions are available. Yearly: $14.99, billed.' - ); - expect(nextAnchor.textContent).toBe(' kept'); }); - it('should restore line breaks between translated syntax-highlighted lines', () => { - globalThis.document.body.innerHTML = [ - '
- [x] This is a complete item\n',
-            '- [ ] This is an incomplete item\n
', - ].join(''); + it.each([ + ['wrapped text before a link', 'Until a repository transfers, keep using\nits location in the ', + 'Until a repository transfers, keep using\nits location in the', + 'Until a repository transfers, keep using\nits location in the '], + ['wrapped text after code', ' are planned to become forwarding pages\nthat guide visitors.', + 'are planned to become forwarding pages\nthat guide visitors.', + ' are planned to become forwarding pages\nthat guide visitors.'], + ['wrapped text between links', ', and\n', ', and', ', and\n'], + ['translated text', ' Read the\nproject notes ', ' Projektberichte lesen ', ' Projektberichte lesen '], + ['nonbreaking whitespace', '\u00a0Read the\nnotes\t', 'Hinweise lesen', '\u00a0Hinweise lesen\t'], + ['unspaced punctuation', '.', '.', '.'], + ['unspaced text', 'Notes', 'Hinweise', 'Hinweise'], + ['whitespace only', ' \n', '', ' \n'], + ])('should preserve source boundaries for %s', (label, source, translation, expected) => { + initCrowdIn(); + jest.runAllTimers(); + const processor = globalThis.i18nextify.i18next.use.mock.calls[0][0]; + expect(processor.process(translation, [source.trim()], { defaultValue: source })).toBe(expected); + }); + + it.each([ + [undefined, {}], + ['Text', undefined], + [null, { defaultValue: ' Text ' }], + [42, { defaultValue: ' Text ' }], + ['Text', {}], + ['Text', { defaultValue: null }], + ])('should leave non-text translations or missing source text unchanged (%j, %j)', (value, options) => { initCrowdIn(); jest.runAllTimers(); - const options = globalThis.proxyTranslator.init.mock.calls[0][0]; - const codeBlock = globalThis.document.getElementById('code-block'); - const firstLineText = codeBlock.querySelector('.k').nextSibling; + const processor = globalThis.i18nextify.i18next.use.mock.calls[0][0]; + expect(processor.process(value, ['key'], options)).toBe(value); + }); - firstLineText.data = firstLineText.data.trim(); - options.callback(); + it.each([ + [{}, ['crowdin-preserve-whitespace']], + [{ postProcess: 'existing' }, ['existing', 'crowdin-preserve-whitespace']], + [{ postProcess: ['first', 'second'] }, ['first', 'second', 'crowdin-preserve-whitespace']], + ])('should append whitespace processing to renderer options %j', (options, expected) => { + const renderer = globalThis.i18nextify; + const originalInit = renderer.init; + const result = { start: jest.fn() }; + const callback = jest.fn(); + originalInit.mockReturnValue(result); + const originalOptions = { ...options, lng: 'de' }; - expect(codeBlock.textContent).toBe( - '- [x] This is a complete item\n- [ ] This is an incomplete item\n' - ); + initCrowdIn(); + jest.runAllTimers(); + + expect(renderer.init(originalOptions, callback)).toBe(result); + expect(originalInit).toHaveBeenCalledWith({ ...originalOptions, postProcess: expected }, callback); + expect(originalInit.mock.contexts[0]).toBe(renderer); + expect(originalOptions).toEqual({ ...options, lng: 'de' }); }); - it('should restore whitespace after later asynchronous DOM changes', async () => { - globalThis.document.body.innerHTML = '

Use this link here.

'; + it.each([ + ['fallback text', ', and\n', ', and', ', and\n'], + ['instant translation', ' Read the\nnotes ', ' Hinweise lesen ', ' Hinweise lesen '], + ])('should preserve %s returned after missing-key postprocessing', (label, source, value, expected) => { + const renderer = globalThis.i18nextify; + const originalInit = renderer.init; + const originalHandler = jest.fn().mockReturnValue(value); + const options = { parseMissingKeyHandler: originalHandler }; + const translationOptions = { defaultValue: source }; initCrowdIn(); jest.runAllTimers(); + renderer.init(options); + const wrappedOptions = originalInit.mock.calls[0][0]; - const options = globalThis.proxyTranslator.init.mock.calls[0][0]; - const translated = globalThis.document.getElementById('translated'); - const link = translated.querySelector('a'); + expect(wrappedOptions.parseMissingKeyHandler(source.trim(), source, translationOptions)).toBe(expected); + expect(originalHandler).toHaveBeenCalledWith(source.trim(), source, translationOptions); + expect(originalHandler.mock.contexts[0]).toBe(wrappedOptions); + expect(options.parseMissingKeyHandler).toBe(originalHandler); + }); - options.callback(); - link.previousSibling.data = link.previousSibling.data.trimEnd(); - await Promise.resolve(); + it('should avoid the whitespace observer loop that froze Doxygen pages', async () => { + document.body.innerHTML = '
code example\n
'; + const observer = jest.spyOn(globalThis, 'MutationObserver'); - expect(translated.textContent).toBe('Use this link here.'); + try { + initCrowdIn('LizardByte-docs', 'dockle'); + jest.advanceTimersByTime(0); + const html = document.body.innerHTML; + const processor = globalThis.i18nextify.i18next.use.mock.calls[0][0]; - jest.advanceTimersByTime(1000); - link.nextSibling.data = link.nextSibling.data.trimStart(); - await Promise.resolve(); + for (let i = 0; i < 10; i++) { + expect(processor.process('Translated', ['Text'], { defaultValue: ' Text\n' })).toBe(' Translated\n'); + await Promise.resolve(); + } + jest.advanceTimersByTime(5000); - expect(translated.textContent).toBe('Use this link here.'); + expect(observer).not.toHaveBeenCalled(); + expect(document.body.innerHTML).toBe(html); + expect(jest.getTimerCount()).toBe(0); + } finally { + observer.mockRestore(); + } }); - it('should not fight another observer over translated whitespace', async () => { - globalThis.document.body.innerHTML = '

Use this link here.

'; - + it('should configure each Crowdin renderer only once', () => { initCrowdIn(); jest.runAllTimers(); + const wrappedInit = globalThis.i18nextify.init; - const options = globalThis.proxyTranslator.init.mock.calls[0][0]; - const text = document.querySelector('a').previousSibling; - options.callback(); - - let rewrites = 0; - const translatorObserver = new MutationObserver(() => { - if (text.data.endsWith(' ') && rewrites < 10) { - rewrites++; - text.data = text.data.trimEnd(); - } - }); - translatorObserver.observe(document.body, { characterData: true, subtree: true }); - - text.data = text.data.trimEnd(); - for (let i = 0; i < 10; i++) await Promise.resolve(); - translatorObserver.disconnect(); + initCrowdIn(); + jest.runAllTimers(); - expect(rewrites).toBe(1); + expect(globalThis.i18nextify.i18next.use).toHaveBeenCalledTimes(1); + expect(globalThis.i18nextify.init).toBe(wrappedInit); }); it('should initialize proxyTranslator with LizardByte-docs settings', () => { @@ -364,6 +386,10 @@ describe('Crowdin fetch interceptor', () => { // Minimal DOM + proxyTranslator stub so initCrowdIn callback doesn't crash globalThis.window.proxyTranslator = { init: jest.fn() }; + globalThis.i18nextify = { + i18next: { use: jest.fn() }, + init: jest.fn(), + }; globalThis.document.body.innerHTML = ''; jest.useFakeTimers(); @@ -374,6 +400,7 @@ describe('Crowdin fetch interceptor', () => { jest.useRealTimers(); delete globalThis._crowdinMirrorInstalled; delete globalThis.window.proxyTranslator; + delete globalThis.i18nextify; }); it('should redirect Crowdin CDN fetch calls to the GitHub Pages mirror', async () => { From 40f58f5fc92da23b73d3d28c01943e8f1235e9e0 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:57:02 -0500 Subject: [PATCH 2/2] docs: revert Crowdin whitespace README addition --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index a293d4e..0f1d4da 100644 --- a/README.md +++ b/README.md @@ -66,18 +66,6 @@ icon filenames that contain spaces or other reserved characters. npm install @lizardbyte/shared-web --ignore-scripts ``` -## CrowdIn whitespace preservation - -The shared initializer preserves the source text's leading and trailing whitespace -inside the translation pipeline, including missing-key fallbacks and instant -translations. This covers multiline text around links, emphasis, and inline code. -It does not install a whitespace repair observer: the renderer caches and displays -the same result, avoiding the observer loop that could freeze documentation pages. - -Crowdin still loads after the page finishes loading. Preformatted code and Doxygen -`.fragment` blocks remain excluded from translation so syntax highlighters can -update them without losing their formatting. - ## CrowdIn in JSDoc Create an initializer such as `docs/crowdin-init.js`: