Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions packages/csv-stringify/lib/sync.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,16 @@ import { normalize_options } from "./api/normalize_options.js";

const stringify = function (records, opts = {}) {
const data = [];
// Join the records by batch, holding many small strings built with `+=`
// until a single final join is slow on large datasets
let batch = [];
const push = function (chunk) {
batch.push(chunk);
if (batch.length === 1000) {
data.push(batch.join(""));
batch = [];
}
};
const [err, options] = normalize_options(opts);
if (err !== undefined) throw err;
const state = {
Expand All @@ -21,20 +31,15 @@ const stringify = function (records, opts = {}) {
};
const api = stringifier(options, state, info);
for (const record of records) {
const err = api.__transform(record, function (record) {
data.push(record);
});
const err = api.__transform(record, push);
if (err !== undefined) throw err;
}
if (data.length === 0) {
api.bom((d) => {
data.push(d);
});
const err = api.headers((headers) => {
data.push(headers);
});
if (data.length === 0 && batch.length === 0) {
api.bom(push);
const err = api.headers(push);
if (err !== undefined) throw err;
}
data.push(batch.join(""));
return data.join("");
};

Expand Down
19 changes: 19 additions & 0 deletions packages/csv-stringify/test/api.sync.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,25 @@ describe("sync", function () {
data.should.eql('"1","2"\n"3","4"');
});

it("work on small and large datasets", function () {
for (const length of [1, 998, 999, 1000, 2500]) {
const records = Array.from({ length }, (_, i) => [i, `a,${i}`]);
const data = stringify(records, {
bom: true,
header: true,
columns: ["a", "b"],
});
data.should.eql(
"a,b\n" + records.map(([i]) => `${i},"a,${i}"\n`).join(""),
);
}
});

it("emit bom and header without records", function () {
const data = stringify([], { bom: true, header: true, columns: ["a"] });
data.should.eql("a\n");
});

it("catch error", function () {
(() => {
stringify([1, { a: "3", b: "4" }]);
Expand Down