From 7a56d7096d8da984141c346bead7b466d4f3a741 Mon Sep 17 00:00:00 2001 From: deadstrobe5 Date: Tue, 10 Nov 2026 03:19:25 +0000 Subject: [PATCH] perf(csv-stringify): join the sync output in batches --- packages/csv-stringify/lib/sync.js | 25 +++++++++++++++---------- packages/csv-stringify/test/api.sync.js | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/packages/csv-stringify/lib/sync.js b/packages/csv-stringify/lib/sync.js index b9e3d10aa..59271509a 100644 --- a/packages/csv-stringify/lib/sync.js +++ b/packages/csv-stringify/lib/sync.js @@ -10,6 +10,16 @@ import { normalize_options } from "./api/normalize_options.js"; const stringify = function (records, opts = {}) { const data = []; + // Join the records by batch, holding many small strings built with `+=` + // until a single final join is slow on large datasets + let batch = []; + const push = function (chunk) { + batch.push(chunk); + if (batch.length === 1000) { + data.push(batch.join("")); + batch = []; + } + }; const [err, options] = normalize_options(opts); if (err !== undefined) throw err; const state = { @@ -21,20 +31,15 @@ const stringify = function (records, opts = {}) { }; const api = stringifier(options, state, info); for (const record of records) { - const err = api.__transform(record, function (record) { - data.push(record); - }); + const err = api.__transform(record, push); if (err !== undefined) throw err; } - if (data.length === 0) { - api.bom((d) => { - data.push(d); - }); - const err = api.headers((headers) => { - data.push(headers); - }); + if (data.length === 0 && batch.length === 0) { + api.bom(push); + const err = api.headers(push); if (err !== undefined) throw err; } + data.push(batch.join("")); return data.join(""); }; diff --git a/packages/csv-stringify/test/api.sync.js b/packages/csv-stringify/test/api.sync.js index 2a97b1206..4af11398e 100644 --- a/packages/csv-stringify/test/api.sync.js +++ b/packages/csv-stringify/test/api.sync.js @@ -29,6 +29,25 @@ describe("sync", function () { data.should.eql('"1","2"\n"3","4"'); }); + it("work on small and large datasets", function () { + for (const length of [1, 998, 999, 1000, 2500]) { + const records = Array.from({ length }, (_, i) => [i, `a,${i}`]); + const data = stringify(records, { + bom: true, + header: true, + columns: ["a", "b"], + }); + data.should.eql( + "a,b\n" + records.map(([i]) => `${i},"a,${i}"\n`).join(""), + ); + } + }); + + it("emit bom and header without records", function () { + const data = stringify([], { bom: true, header: true, columns: ["a"] }); + data.should.eql("a\n"); + }); + it("catch error", function () { (() => { stringify([1, { a: "3", b: "4" }]);