Skip to content

Commit d0cd7b1

Browse files
risenWrisenW
committed
Fixes #354 
1 parent b2a14df commit d0cd7b1

File tree

5 files changed

+182
-3
lines changed

5 files changed

+182
-3
lines changed

‎src/danfojs-base/io/browser/io.csv.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ const $readCSV = async (file: any, options?: CsvInputOptionsBrowser): Promise<Da
5151
return new Promise(resolve => {
5252
Papa.parse(file, {
5353
header: true,
54+
dynamicTyping: true,
5455
...options,
5556
download: true,
5657
complete: results => {
@@ -83,6 +84,7 @@ const $streamCSV = async (file: string, callback: (df: DataFrame) => void, optio
8384
let count = -1
8485
Papa.parse(file, {
8586
...options,
87+
dynamicTyping: true,
8688
header: true,
8789
download: true,
8890
step: results => {

‎src/danfojs-base/io/node/io.csv.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ const $readCSV = async (filePath: string, options?: CsvInputOptionsNode): Promis
5454
return new Promise(resolve => {
5555
const optionsWithDefaults = {
5656
header: true,
57+
dynamicTyping: true,
5758
...options,
5859
}
5960

@@ -76,6 +77,7 @@ const $readCSV = async (filePath: string, options?: CsvInputOptionsNode): Promis
7677
const fileStream = fs.createReadStream(filePath)
7778
Papa.parse(fileStream, {
7879
header: true,
80+
dynamicTyping: true,
7981
...options,
8082
complete: results => {
8183
const df = new DataFrame(results.data, frameConfig);
@@ -107,6 +109,7 @@ const $streamCSV = async (filePath: string, callback: (df: DataFrame) => void, o
107109
if (filePath.startsWith("http") || filePath.startsWith("https")) {
108110
const optionsWithDefaults = {
109111
header: true,
112+
dynamicTyping: true,
110113
...options,
111114
}
112115
return new Promise(resolve => {
@@ -132,6 +135,7 @@ const $streamCSV = async (filePath: string, callback: (df: DataFrame) => void, o
132135
let count = -1
133136
Papa.parse(fileStream, {
134137
header: true,
138+
dynamicTyping: true,
135139
...options,
136140
step: results => {
137141
const df = new DataFrame([results.data], { ...frameConfig, index: [count++] });
@@ -223,7 +227,7 @@ const $openCsvInputStream = (filePath: string, options: CsvInputOptionsNode) =>
223227

224228
if (filePath.startsWith("http") || filePath.startsWith("https")) {
225229
const dataStream = request.get(filePath);
226-
const parseStream: any = Papa.parse(Papa.NODE_STREAM_INPUT, { header, ...options });
230+
const parseStream: any = Papa.parse(Papa.NODE_STREAM_INPUT, { header, dynamicTyping: true, ...options });
227231
dataStream.pipe(parseStream);
228232
let count = -1
229233

@@ -255,7 +259,7 @@ const $openCsvInputStream = (filePath: string, options: CsvInputOptionsNode) =>
255259
const fileStream = fs.createReadStream(filePath)
256260
let count = -1
257261
Papa.parse(fileStream, {
258-
...{ header, ...options },
262+
...{ header, dynamicTyping: true, ...options },
259263
step: results => {
260264
if (isFirstChunk) {
261265
if (header === true) {

‎src/danfojs-browser/tests/io/csv.reader.test.js‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ describe("readCSV", function () {
5858
]);
5959
});
6060

61+
it("Read remote csv file works and returns correct data type", async function () {
62+
const remoteFile = "https://raw.githubusercontent.com/javascriptdata/danfojs/dev/src/danfojs-node/test/samples/titanic.csv";
63+
let df = await dfd.readCSV(remoteFile, { header: true, preview: 2 });
64+
const values = [
65+
[ 0, 3, 'Mr. Owen Harris Braund', 'male', 22, 1, 0, 7.25 ],
66+
[ 1, 1, 'Mrs. John Bradley (Florence Briggs Thayer) Cumings', 'female', 38, 1, 0, 71.2833 ]
67+
];
68+
assert.deepEqual(df.values, values);
69+
});
70+
6171
});
6272

6373
// describe("streamCSV", function () {

‎src/danfojs-node/test/io/csv.reader.test.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,19 @@ describe("readCSV", function () {
5959
'int32', 'float32'
6060
]);
6161
});
62-
62+
it("Read local csv with correct types and format works", async function () {
63+
const filePath = path.join(process.cwd(), "test", "samples", "iris.csv");
64+
let df: any = await readCSV(filePath, { header: true, preview: 5 });
65+
const values = [
66+
[5.1, 3.5, 1.4, 0.2, 0.0],
67+
[4.9, 3.0, 1.4, 0.2, 0.0],
68+
[4.7, 3.2, 1.3, 0.2, 0.0],
69+
[4.6, 3.1, 1.5, 0.2, 0.0],
70+
[5.0, 3.6, 1.4, 0.2, 0.0],
71+
]
72+
console.log(df.values)
73+
assert.deepEqual(df.values, values);
74+
});
6375
// it("Read remote csv file works", async function () {
6476
// const remoteFile = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/titanic.csv"
6577
// let df: any = await readCSV(remoteFile, { header: true, preview: 5 });

‎src/danfojs-node/test/samples/iris.csv‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
sepal length (cm),sepal width (cm),petal length (cm),petal width (cm),target
2+
5.1,3.5,1.4,0.2,0.0
3+
4.9,3.0,1.4,0.2,0.0
4+
4.7,3.2,1.3,0.2,0.0
5+
4.6,3.1,1.5,0.2,0.0
6+
5.0,3.6,1.4,0.2,0.0
7+
5.4,3.9,1.7,0.4,0.0
8+
4.6,3.4,1.4,0.3,0.0
9+
5.0,3.4,1.5,0.2,0.0
10+
4.4,2.9,1.4,0.2,0.0
11+
4.9,3.1,1.5,0.1,0.0
12+
5.4,3.7,1.5,0.2,0.0
13+
4.8,3.4,1.6,0.2,0.0
14+
4.8,3.0,1.4,0.1,0.0
15+
4.3,3.0,1.1,0.1,0.0
16+
5.8,4.0,1.2,0.2,0.0
17+
5.7,4.4,1.5,0.4,0.0
18+
5.4,3.9,1.3,0.4,0.0
19+
5.1,3.5,1.4,0.3,0.0
20+
5.7,3.8,1.7,0.3,0.0
21+
5.1,3.8,1.5,0.3,0.0
22+
5.4,3.4,1.7,0.2,0.0
23+
5.1,3.7,1.5,0.4,0.0
24+
4.6,3.6,1.0,0.2,0.0
25+
5.1,3.3,1.7,0.5,0.0
26+
4.8,3.4,1.9,0.2,0.0
27+
5.0,3.0,1.6,0.2,0.0
28+
5.0,3.4,1.6,0.4,0.0
29+
5.2,3.5,1.5,0.2,0.0
30+
5.2,3.4,1.4,0.2,0.0
31+
4.7,3.2,1.6,0.2,0.0
32+
4.8,3.1,1.6,0.2,0.0
33+
5.4,3.4,1.5,0.4,0.0
34+
5.2,4.1,1.5,0.1,0.0
35+
5.5,4.2,1.4,0.2,0.0
36+
4.9,3.1,1.5,0.2,0.0
37+
5.0,3.2,1.2,0.2,0.0
38+
5.5,3.5,1.3,0.2,0.0
39+
4.9,3.6,1.4,0.1,0.0
40+
4.4,3.0,1.3,0.2,0.0
41+
5.1,3.4,1.5,0.2,0.0
42+
5.0,3.5,1.3,0.3,0.0
43+
4.5,2.3,1.3,0.3,0.0
44+
4.4,3.2,1.3,0.2,0.0
45+
5.0,3.5,1.6,0.6,0.0
46+
5.1,3.8,1.9,0.4,0.0
47+
4.8,3.0,1.4,0.3,0.0
48+
5.1,3.8,1.6,0.2,0.0
49+
4.6,3.2,1.4,0.2,0.0
50+
5.3,3.7,1.5,0.2,0.0
51+
5.0,3.3,1.4,0.2,0.0
52+
7.0,3.2,4.7,1.4,1.0
53+
6.4,3.2,4.5,1.5,1.0
54+
6.9,3.1,4.9,1.5,1.0
55+
5.5,2.3,4.0,1.3,1.0
56+
6.5,2.8,4.6,1.5,1.0
57+
5.7,2.8,4.5,1.3,1.0
58+
6.3,3.3,4.7,1.6,1.0
59+
4.9,2.4,3.3,1.0,1.0
60+
6.6,2.9,4.6,1.3,1.0
61+
5.2,2.7,3.9,1.4,1.0
62+
5.0,2.0,3.5,1.0,1.0
63+
5.9,3.0,4.2,1.5,1.0
64+
6.0,2.2,4.0,1.0,1.0
65+
6.1,2.9,4.7,1.4,1.0
66+
5.6,2.9,3.6,1.3,1.0
67+
6.7,3.1,4.4,1.4,1.0
68+
5.6,3.0,4.5,1.5,1.0
69+
5.8,2.7,4.1,1.0,1.0
70+
6.2,2.2,4.5,1.5,1.0
71+
5.6,2.5,3.9,1.1,1.0
72+
5.9,3.2,4.8,1.8,1.0
73+
6.1,2.8,4.0,1.3,1.0
74+
6.3,2.5,4.9,1.5,1.0
75+
6.1,2.8,4.7,1.2,1.0
76+
6.4,2.9,4.3,1.3,1.0
77+
6.6,3.0,4.4,1.4,1.0
78+
6.8,2.8,4.8,1.4,1.0
79+
6.7,3.0,5.0,1.7,1.0
80+
6.0,2.9,4.5,1.5,1.0
81+
5.7,2.6,3.5,1.0,1.0
82+
5.5,2.4,3.8,1.1,1.0
83+
5.5,2.4,3.7,1.0,1.0
84+
5.8,2.7,3.9,1.2,1.0
85+
6.0,2.7,5.1,1.6,1.0
86+
5.4,3.0,4.5,1.5,1.0
87+
6.0,3.4,4.5,1.6,1.0
88+
6.7,3.1,4.7,1.5,1.0
89+
6.3,2.3,4.4,1.3,1.0
90+
5.6,3.0,4.1,1.3,1.0
91+
5.5,2.5,4.0,1.3,1.0
92+
5.5,2.6,4.4,1.2,1.0
93+
6.1,3.0,4.6,1.4,1.0
94+
5.8,2.6,4.0,1.2,1.0
95+
5.0,2.3,3.3,1.0,1.0
96+
5.6,2.7,4.2,1.3,1.0
97+
5.7,3.0,4.2,1.2,1.0
98+
5.7,2.9,4.2,1.3,1.0
99+
6.2,2.9,4.3,1.3,1.0
100+
5.1,2.5,3.0,1.1,1.0
101+
5.7,2.8,4.1,1.3,1.0
102+
6.3,3.3,6.0,2.5,2.0
103+
5.8,2.7,5.1,1.9,2.0
104+
7.1,3.0,5.9,2.1,2.0
105+
6.3,2.9,5.6,1.8,2.0
106+
6.5,3.0,5.8,2.2,2.0
107+
7.6,3.0,6.6,2.1,2.0
108+
4.9,2.5,4.5,1.7,2.0
109+
7.3,2.9,6.3,1.8,2.0
110+
6.7,2.5,5.8,1.8,2.0
111+
7.2,3.6,6.1,2.5,2.0
112+
6.5,3.2,5.1,2.0,2.0
113+
6.4,2.7,5.3,1.9,2.0
114+
6.8,3.0,5.5,2.1,2.0
115+
5.7,2.5,5.0,2.0,2.0
116+
5.8,2.8,5.1,2.4,2.0
117+
6.4,3.2,5.3,2.3,2.0
118+
6.5,3.0,5.5,1.8,2.0
119+
7.7,3.8,6.7,2.2,2.0
120+
7.7,2.6,6.9,2.3,2.0
121+
6.0,2.2,5.0,1.5,2.0
122+
6.9,3.2,5.7,2.3,2.0
123+
5.6,2.8,4.9,2.0,2.0
124+
7.7,2.8,6.7,2.0,2.0
125+
6.3,2.7,4.9,1.8,2.0
126+
6.7,3.3,5.7,2.1,2.0
127+
7.2,3.2,6.0,1.8,2.0
128+
6.2,2.8,4.8,1.8,2.0
129+
6.1,3.0,4.9,1.8,2.0
130+
6.4,2.8,5.6,2.1,2.0
131+
7.2,3.0,5.8,1.6,2.0
132+
7.4,2.8,6.1,1.9,2.0
133+
7.9,3.8,6.4,2.0,2.0
134+
6.4,2.8,5.6,2.2,2.0
135+
6.3,2.8,5.1,1.5,2.0
136+
6.1,2.6,5.6,1.4,2.0
137+
7.7,3.0,6.1,2.3,2.0
138+
6.3,3.4,5.6,2.4,2.0
139+
6.4,3.1,5.5,1.8,2.0
140+
6.0,3.0,4.8,1.8,2.0
141+
6.9,3.1,5.4,2.1,2.0
142+
6.7,3.1,5.6,2.4,2.0
143+
6.9,3.1,5.1,2.3,2.0
144+
5.8,2.7,5.1,1.9,2.0
145+
6.8,3.2,5.9,2.3,2.0
146+
6.7,3.3,5.7,2.5,2.0
147+
6.7,3.0,5.2,2.3,2.0
148+
6.3,2.5,5.0,1.9,2.0
149+
6.5,3.0,5.2,2.0,2.0
150+
6.2,3.4,5.4,2.3,2.0
151+
5.9,3.0,5.1,1.8,2.0

0 commit comments

Comments
 (0)