Skip to content

Commit 0d7b815

Browse files
mbostockmbostock
authored
FileAttachment.csv({typed: "auto"}) (#169)
* inferSchema, enforceSchema * checkpoint tests * rewrite everything * even simpler * distinguish invalid vs. empty boolean * typed: "auto" * fix time zones * remove pointless capturing groups * document DATE_TEST better
1 parent 1c8e9dc commit 0d7b815

4 files changed

Lines changed: 172 additions & 5 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"version": "2.1.7",
99
"type": "module",
1010
"scripts": {
11-
"test": "vitest",
11+
"test": "TZ=America/Los_Angeles vitest",
1212
"prepublishOnly": "rm -rf dist && tsc && chmod +x dist/bin/*.js && cp -r src/styles src/templates dist/src && cp src/runtime/stdlib/*.css dist/src/runtime/stdlib",
1313
"lint": "tsc --noEmit && eslint bin src types",
1414
"notebooks": "tsx bin/notebooks.ts",

‎src/runtime/stdlib/dsv.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import {assert, describe, it} from "vitest";
2+
import {inferTypes} from "./dsv";
3+
import {coerceBoolean, coerceDate, coerceNumber} from "./dsv";
4+
5+
function inferType(values: string[]): unknown[] {
6+
return inferTypes(values.map((value) => ({value})), ["value"]).map(({value}) => value); // prettier-ignore
7+
}
8+
9+
describe("inferTypes", () => {
10+
it("infers numbers", () => {
11+
assert.deepStrictEqual(inferType(["2", "4", "6"]), [2, 4, 6]);
12+
assert.deepStrictEqual(inferType(["1.2", "3.4", "5.67"]), [1.2, 3.4, 5.67]);
13+
assert.deepStrictEqual(inferType([".2", ".4", ".67"]), [0.2, 0.4, 0.67]);
14+
assert.deepStrictEqual(inferType([0.1, 1, 2, 3, 9].map(String)), [0.1, 1, 2, 3, 9]);
15+
assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 1, 3, 2, 3].map(String)), [NaN, 1, 2, 3, 9, 1, 1, 1, 3, 2, 3]); // prettier-ignore
16+
assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", ".9", "x"]), [0, 0.1, 2, 3, 4, 5, 2, 3, 0.9, NaN]); // prettier-ignore
17+
});
18+
it("infers booleans", () => {
19+
assert.deepStrictEqual(inferType(["true", "", "false"]), [true, null, false]);
20+
assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, "", "pants on fire"].map(String)), [true, false, true, false, true, false, true, false, true, null, undefined]); // prettier-ignore
21+
});
22+
it("infers dates in common formats", () => {
23+
assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), [new Date("2020-01-02T08:00:00.000Z"), new Date("2020-11-12T20:23:00.000Z"), null, new Date("2020-01-12T00:00:00.000Z")]); // prettier-ignore
24+
});
25+
it("infers strings", () => {
26+
assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), ["cat", "dog", "1,000", "null"]); // prettier-ignore
27+
assert.deepStrictEqual(inferType(["10n", "22n", "0n"]), ["10n", "22n", "0n"]);
28+
assert.deepStrictEqual(inferType(["", ""]), ["", ""]);
29+
assert.deepStrictEqual(inferType(["NaN", "NaN", "NaN", "1", "2", "3"]), ["NaN", "NaN", "NaN", "1", "2", "3"]); // prettier-ignore
30+
assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 3].map(String)), ["x", 1, 2, 3, 9, 1, 1, 3].map(String)); // prettier-ignore
31+
assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", "x"]), [".0", ".1", "2", "3", "4", "5", "2", "3", "x"]); // prettier-ignore
32+
assert.deepStrictEqual(inferType([true, false, true, false, true, false, "pants on fire"].map(String)), ["true", "false", "true", "false", "true", "false", "pants on fire"]); // prettier-ignore
33+
});
34+
});
35+
36+
describe("coerceNumber", () => {
37+
it("coerces to number", () => {
38+
assert.deepStrictEqual(coerceNumber("1.2"), 1.2);
39+
assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2);
40+
assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2);
41+
assert.deepStrictEqual(coerceNumber("10"), 10);
42+
assert.deepStrictEqual(coerceNumber("0"), 0);
43+
assert.deepStrictEqual(coerceNumber("A"), NaN);
44+
assert.deepStrictEqual(coerceNumber(""), NaN);
45+
assert.deepStrictEqual(coerceNumber(" "), NaN);
46+
assert.deepStrictEqual(coerceNumber("null"), NaN);
47+
assert.deepStrictEqual(coerceNumber("1.2"), 1.2);
48+
assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2);
49+
assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2);
50+
assert.deepStrictEqual(coerceNumber("0"), 0);
51+
assert.deepStrictEqual(coerceNumber(""), NaN);
52+
assert.deepStrictEqual(coerceNumber(" "), NaN);
53+
assert.deepStrictEqual(coerceNumber("A"), NaN);
54+
assert.deepStrictEqual(coerceNumber("null"), NaN);
55+
assert.deepStrictEqual(coerceNumber("undefined"), NaN);
56+
assert.deepStrictEqual(coerceNumber("{a: 1}"), NaN);
57+
});
58+
});
59+
60+
describe("coerceBoolean", () => {
61+
it("coerces to boolean", () => {
62+
assert.deepStrictEqual(coerceBoolean("true"), true);
63+
assert.deepStrictEqual(coerceBoolean("True "), true);
64+
assert.deepStrictEqual(coerceBoolean("False"), false);
65+
assert.deepStrictEqual(coerceBoolean("false"), false);
66+
assert.deepStrictEqual(coerceBoolean(""), null);
67+
assert.deepStrictEqual(coerceBoolean("1"), undefined);
68+
assert.deepStrictEqual(coerceBoolean("2"), undefined);
69+
assert.deepStrictEqual(coerceBoolean("0"), undefined);
70+
assert.deepStrictEqual(coerceBoolean("null"), undefined);
71+
assert.deepStrictEqual(coerceBoolean("undefined"), undefined);
72+
});
73+
});
74+
75+
describe("coerceDate", () => {
76+
it("coerces to date", () => {
77+
const invalidDate = new Date(NaN);
78+
assert.deepStrictEqual(coerceDate("12/12/2020"), new Date("12/12/2020"));
79+
assert.deepStrictEqual(coerceDate("12/12/2020 "), new Date("12/12/2020"));
80+
assert.deepStrictEqual(coerceDate("2022-01-01T12:34:00Z"), new Date("2022-01-01T12:34:00Z")); // prettier-ignore
81+
assert.deepStrictEqual(coerceDate("{a: 1}"), invalidDate);
82+
assert.deepStrictEqual(coerceDate("true"), invalidDate);
83+
assert.deepStrictEqual(coerceDate("2020-1-12"), invalidDate);
84+
assert.deepStrictEqual(coerceDate("1675356739000"), invalidDate);
85+
assert.deepStrictEqual(coerceDate("undefined"), invalidDate);
86+
assert.deepStrictEqual(coerceDate(""), null);
87+
});
88+
});

‎src/runtime/stdlib/dsv.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/* eslint-disable @typescript-eslint/no-unused-expressions */
2+
3+
/**
4+
* Accepts dates in the following forms:
5+
* - ±YYYYYY-MM-DD
6+
* - YYYY-MM-DD
7+
* - MM/DD/YY
8+
* - MM/DD/YYYY
9+
*
10+
* Following a "T" or a space, dates may additionally have a time:
11+
* - HH:MM
12+
* - HH:MM:SS
13+
* - HH:MM:SS.MMM
14+
*
15+
* And the time may optionally have a time zone:
16+
* - Z
17+
* - ±HH:MM
18+
*/
19+
const DATE_TEST = /^([-+]\d{2})?\d{4}-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/\d{2,4}([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore
20+
21+
/** The maximum number of rows to sample (including missing values). */
22+
const SAMPLE_SIZE = 100;
23+
24+
export function inferTypes<T extends Record<string, string>>(
25+
rows: T[],
26+
columns: (keyof T)[]
27+
): Record<keyof T, unknown>[] {
28+
const output = rows as Record<keyof T, unknown>[];
29+
const n = rows.length;
30+
const k = Math.min(n, SAMPLE_SIZE);
31+
for (const column of columns) {
32+
let booleans = 0;
33+
let numbers = 0;
34+
let dates = 0;
35+
let strings = 0;
36+
37+
for (let i = 0; i < k; ++i) {
38+
const value = rows[i][column]?.trim();
39+
if (!value) continue;
40+
++strings;
41+
if (/^(true|false)$/i.test(value)) ++booleans;
42+
else if (!isNaN(Number(value))) ++numbers;
43+
else if (DATE_TEST.test(value)) ++dates;
44+
}
45+
46+
// Chose the non-string type with the greatest count that is also ≥90%; or
47+
// if no such type meets that criterion, use string.
48+
let coerce: ((value: string) => unknown) | undefined = undefined;
49+
let typeCount = Math.max(1, Math.ceil(strings * 0.9)) - 1;
50+
if (booleans > typeCount) ((coerce = coerceBoolean), (typeCount = booleans));
51+
if (numbers > typeCount) ((coerce = coerceNumber), (typeCount = numbers));
52+
if (dates > typeCount) ((coerce = coerceDate), (typeCount = dates));
53+
if (!coerce) continue;
54+
55+
for (let i = 0; i < n; ++i) {
56+
output[i][column] = coerce(rows[i][column]);
57+
}
58+
}
59+
return output;
60+
}
61+
62+
export function coerceBoolean(value: string): boolean | null | undefined {
63+
const trimmed = value.trim().toLowerCase();
64+
return trimmed === "true" ? true : trimmed === "false" ? false : trimmed ? undefined : null;
65+
}
66+
67+
export function coerceNumber(value: string): number {
68+
const trimmed = value.trim();
69+
return trimmed ? Number(value) : NaN;
70+
}
71+
72+
export function coerceDate(value: string): Date | null {
73+
const trimmed = value.trim();
74+
return trimmed ? new Date(DATE_TEST.test(trimmed) ? trimmed : NaN) : null;
75+
}

‎src/runtime/stdlib/fileAttachment.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1+
import {inferTypes} from "./dsv.js";
2+
13
/* eslint-disable @typescript-eslint/no-explicit-any */
24
const files = new Map<string, FileAttachmentImpl>();
35

4-
export type DsvOptions = {delimiter?: string; array?: boolean; typed?: boolean};
6+
export type DsvOptions = {delimiter?: string; array?: boolean; typed?: "auto" | boolean};
57
export type DsvResult = (Record<string, any>[] | any[][]) & {columns: string[]};
68

79
export interface FileAttachment {
@@ -136,11 +138,13 @@ export abstract class AbstractFile implements FileAttachment {
136138
async stream(): Promise<ReadableStream<Uint8Array<ArrayBufferLike>>> {
137139
return (await fetchFile(this)).body!;
138140
}
139-
async dsv({delimiter = ",", array = false, typed = false} = {}): Promise<DsvResult> {
141+
async dsv({delimiter = ",", array = false, typed = false}: DsvOptions = {}): Promise<DsvResult> {
140142
const [text, d3] = await Promise.all([this.text(), import("https://cdn.jsdelivr.net/npm/d3-dsv/+esm")]); // prettier-ignore
141143
const format = d3.dsvFormat(delimiter);
142-
const parse = array ? format.parseRows : format.parse;
143-
return parse(text, typed && d3.autoType);
144+
const parse = array ? ((typed &&= true), format.parseRows) : format.parse;
145+
const output = parse(text, typed === true ? d3.autoType : undefined);
146+
if (typed === "auto") inferTypes(output, output.columns);
147+
return output;
144148
}
145149
async csv(options?: Omit<DsvOptions, "delimiter">): Promise<DsvResult> {
146150
return this.dsv({...options, delimiter: ","});

0 commit comments

Comments
 (0)