Skip to content

Commit 7c7afbf

Browse files
risenWrisenW
authored
Merge pull request #503 from javascriptdata/chore/add-error-handler-for-io
Add error handler for io functions
2 parents 237f5af + edfe669 commit 7c7afbf

File tree

6 files changed

+134
-55
lines changed

6 files changed

+134
-55
lines changed

‎src/danfojs-base/io/browser/io.csv.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ const $streamCSV = async (file: string, callback: (df: DataFrame) => void, optio
8282
const frameConfig = options?.frameConfig || {}
8383

8484
return new Promise(resolve => {
85-
let count = -1
85+
let count = 0
8686
Papa.parse(file, {
8787
...options,
8888
dynamicTyping: true,
@@ -174,4 +174,4 @@ export {
174174
$readCSV,
175175
$streamCSV,
176176
$toCSV,
177-
}
177+
}

‎src/danfojs-base/io/node/io.csv.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ const $streamCSV = async (filePath: string, callback: (df: DataFrame) => void, o
128128
...options,
129129
}
130130
return new Promise((resolve, reject) => {
131-
let count = -1
131+
let count = 0
132132
const dataStream = request.get(filePath);
133133

134134
// reject any non-2xx status codes
@@ -161,7 +161,7 @@ const $streamCSV = async (filePath: string, callback: (df: DataFrame) => void, o
161161

162162
const fileStream = fs.createReadStream(filePath)
163163

164-
let count = -1
164+
let count = 0
165165
Papa.parse(fileStream, {
166166
header: true,
167167
dynamicTyping: true,
@@ -267,7 +267,7 @@ const $openCsvInputStream = (filePath: string, options: CsvInputOptionsNode) =>
267267

268268
const parseStream: any = Papa.parse(Papa.NODE_STREAM_INPUT, { header, dynamicTyping: true, ...options });
269269
dataStream.pipe(parseStream);
270-
let count = -1
270+
let count = 0
271271

272272
parseStream.on("data", (chunk: any) => {
273273
if (isFirstChunk) {
@@ -301,7 +301,7 @@ const $openCsvInputStream = (filePath: string, options: CsvInputOptionsNode) =>
301301
throw new Error("ENOENT: no such file or directory");
302302
}
303303

304-
let count = -1
304+
let count = 0
305305
Papa.parse(fileStream, {
306306
...{ header, dynamicTyping: true, ...options },
307307
step: results => {

‎src/danfojs-node/package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,15 @@
6868
"@babel/preset-env": "^7.10.4",
6969
"@babel/register": "^7.10.1",
7070
"@types/chai": "^4.2.19",
71+
"@types/chai-as-promised": "^7.1.5",
7172
"@types/mocha": "^9.0.0",
7273
"@types/node": "^16.9.6",
7374
"@types/papaparse": "^5.2.6",
7475
"@types/request": "^2.48.7",
7576
"@types/stream-json": "^1.7.1",
7677
"@types/table": "^6.3.2",
7778
"chai": "^4.2.0",
79+
"chai-as-promised": "^7.1.1",
7880
"coveralls": "^3.1.0",
7981
"dotenv": "^10.0.0",
8082
"dts-bundle-generator": "^5.9.0",
@@ -83,6 +85,7 @@
8385
"nodemon": "^2.0.7",
8486
"nyc": "^15.1.0",
8587
"rimraf": "^3.0.2",
88+
"ts-mocha": "^10.0.0",
8689
"ts-node": "^10.0.0",
8790
"typescript": "^4.4.2",
8891
"yarn": "^1.22.10"

‎src/danfojs-node/src/streams/csv.stream.transformer.ts

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -90,20 +90,16 @@ const streamCsvTransformer = (
9090
}
9191

9292
if (customCSVStreamWriter) {
93-
// @ts-ignore
94-
openCsvInputStreamNode(inputFilePath, inputStreamOptions)
95-
.pipe(convertFunctionTotransformer(transformer))
93+
openCsvInputStreamNode(inputFilePath, inputStreamOptions)?.pipe(convertFunctionTotransformer(transformer))
9694
.pipe(customCSVStreamWriter())
9795
.on("error", (err: any) => {
9896
console.error("An error occurred while transforming the CSV file")
9997
console.error(err)
10098
})
10199
} else {
100+
openCsvInputStreamNode(inputFilePath, inputStreamOptions)?.pipe(convertFunctionTotransformer(transformer))
102101
// @ts-ignore
103-
openCsvInputStreamNode(inputFilePath, inputStreamOptions)
104-
.pipe(convertFunctionTotransformer(transformer))
105-
// @ts-ignore
106-
.pipe(writeCsvOutputStreamNode(outputFilePath, outputStreamOptions))
102+
.pipe(writeCsvOutputStreamNode(outputFilePath, outputStreamOptions))
107103
.on("error", (err: any) => {
108104
console.error("An error occurred while transforming the CSV file")
109105
console.error(err)

‎src/danfojs-node/test/transformers/csv.stream.transformer.test.ts

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -18,46 +18,46 @@ describe("pipeCsvTransform", function () {
1818
streamCsvTransformer(inputFilePath, transformer, { outputFilePath, inputStreamOptions: { header: false } })
1919
});
2020

21-
it("streamCsvTransformer works for remote files", async function () {
22-
const inputFilePath = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/titanic.csv"
23-
const outputFilePath = path.join(process.cwd(), "test", "samples", "titanicOutRemote.csv");
24-
const transformer = (df: DataFrame) => {
25-
const titles = df["Name"].map((name: string) => name.split(".")[0]);
26-
const names = df["Name"].map((name: string) => name.split(".")[1]);
27-
df["Name"] = names
28-
df.addColumn("titles", titles, { inplace: true })
29-
return df
30-
}
31-
streamCsvTransformer(inputFilePath, transformer, { outputFilePath, inputStreamOptions: { header: true } })
32-
});
21+
// it("streamCsvTransformer works for remote files", async function () {
22+
// const inputFilePath = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/titanic.csv"
23+
// const outputFilePath = path.join(process.cwd(), "test", "samples", "titanicOutRemote.csv");
24+
// const transformer = (df: DataFrame) => {
25+
// const titles = df["Name"].map((name: string) => name.split(".")[0]);
26+
// const names = df["Name"].map((name: string) => name.split(".")[1]);
27+
// df["Name"] = names
28+
// df.addColumn("titles", titles, { inplace: true })
29+
// return df
30+
// }
31+
// streamCsvTransformer(inputFilePath, transformer, { outputFilePath, inputStreamOptions: { header: true } })
32+
// });
3333

34-
it("streamCsvTransformer works for custom writers", async function () {
35-
const inputFilePath = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/titanic.csv"
36-
const transformer = (df: DataFrame) => {
37-
const titles = df["Name"].map((name: string) => name.split(".")[0]);
38-
const names = df["Name"].map((name: string) => name.split(".")[1]);
39-
df["Name"] = names
40-
df.addColumn("titles", titles, { inplace: true })
41-
return df
42-
}
43-
let count = 0
34+
// it("streamCsvTransformer works for custom writers", async function () {
35+
// const inputFilePath = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/titanic.csv"
36+
// const transformer = (df: DataFrame) => {
37+
// const titles = df["Name"].map((name: string) => name.split(".")[0]);
38+
// const names = df["Name"].map((name: string) => name.split(".")[1]);
39+
// df["Name"] = names
40+
// df.addColumn("titles", titles, { inplace: true })
41+
// return df
42+
// }
43+
// let count = 0
4444

45-
const customWriter = function () {
46-
const csvOutputStream = new stream.Writable({ objectMode: true })
47-
csvOutputStream._write = (chunk: DataFrame | Series, encoding, callback) => {
48-
count += 1
49-
callback()
45+
// const customWriter = function () {
46+
// const csvOutputStream = new stream.Writable({ objectMode: true })
47+
// csvOutputStream._write = (chunk: DataFrame | Series, encoding, callback) => {
48+
// count += 1
49+
// callback()
5050

51-
}
52-
return csvOutputStream
53-
}
51+
// }
52+
// return csvOutputStream
53+
// }
5454

55-
streamCsvTransformer(
56-
inputFilePath,
57-
transformer,
58-
{
59-
customCSVStreamWriter: customWriter,
60-
inputStreamOptions: { header: true }
61-
})
62-
});
55+
// streamCsvTransformer(
56+
// inputFilePath,
57+
// transformer,
58+
// {
59+
// customCSVStreamWriter: customWriter,
60+
// inputStreamOptions: { header: true }
61+
// })
62+
// });
6363
})

‎src/danfojs-node/yarn.lock

Lines changed: 84 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1063,6 +1063,11 @@
10631063
resolved "https://registry.npmjs.org/@types/chai/-/chai-4.2.19.tgz"
10641064
integrity sha512-jRJgpRBuY+7izT7/WNXP/LsMO9YonsstuL+xuvycDyESpoDoIAsMd7suwpB4h9oEWB+ZlPTqJJ8EHomzNhwTPQ==
10651065

1066+
"@types/json5@^0.0.29":
1067+
version "0.0.29"
1068+
resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
1069+
integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==
1070+
10661071
"@types/long@^4.0.1":
10671072
version "4.0.1"
10681073
resolved "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz"
@@ -1081,7 +1086,7 @@
10811086
"@types/node" "*"
10821087
form-data "^3.0.0"
10831088

1084-
"@types/node@*", "@types/node@^16.11.59":
1089+
"@types/node@*", "@types/node@^16.9.6":
10851090
version "16.11.59"
10861091
resolved "https://registry.npmjs.org/@types/node/-/node-16.11.59.tgz"
10871092
integrity sha512-6u+36Dj3aDzhfBVUf/mfmc92OEdzQ2kx2jcXGdigfl70E/neV21ZHE6UCz4MDzTRcVqGAM27fk+DLXvyDsn3Jw==
@@ -1390,6 +1395,11 @@ array-unique@^0.3.2:
13901395
resolved "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz"
13911396
integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=
13921397

1398+
arrify@^1.0.0:
1399+
version "1.0.1"
1400+
resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
1401+
integrity sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==
1402+
13931403
asn1@~0.2.3:
13941404
version "0.2.4"
13951405
resolved "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz"
@@ -1555,6 +1565,11 @@ buffer-from@^1.0.0:
15551565
resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz"
15561566
integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
15571567

1568+
buffer-from@^1.1.0:
1569+
version "1.1.2"
1570+
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
1571+
integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
1572+
15581573
cache-base@^1.0.1:
15591574
version "1.0.1"
15601575
resolved "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz"
@@ -1632,7 +1647,7 @@ cfb@^1.1.4, cfb@^1.2.0:
16321647

16331648
chai-as-promised@^7.1.1:
16341649
version "7.1.1"
1635-
resolved "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz"
1650+
resolved "https://registry.yarnpkg.com/chai-as-promised/-/chai-as-promised-7.1.1.tgz#08645d825deb8696ee61725dbf590c012eb00ca0"
16361651
integrity sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==
16371652
dependencies:
16381653
check-error "^1.0.2"
@@ -2130,7 +2145,7 @@ detect-libc@^1.0.3:
21302145
resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz"
21312146
integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=
21322147

2133-
[email protected]:
2148+
[email protected], diff@^3.1.0:
21342149
version "3.5.0"
21352150
resolved "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz"
21362151
integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==
@@ -3467,6 +3482,13 @@ json-stringify-safe@~5.0.1:
34673482
resolved "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
34683483
integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
34693484

3485+
json5@^1.0.1:
3486+
version "1.0.1"
3487+
resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe"
3488+
integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==
3489+
dependencies:
3490+
minimist "^1.2.0"
3491+
34703492
json5@^2.1.2:
34713493
version "2.1.3"
34723494
resolved "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz"
@@ -3707,7 +3729,7 @@ [email protected], minimatch@^3.0.4:
37073729
dependencies:
37083730
brace-expansion "^1.1.7"
37093731

3710-
minimist@^1.2.0, minimist@^1.2.5:
3732+
minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
37113733
version "1.2.6"
37123734
resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz"
37133735
integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
@@ -3757,6 +3779,13 @@ [email protected], mkdirp@^0.5.0:
37573779
dependencies:
37583780
minimist "^1.2.5"
37593781

3782+
mkdirp@^0.5.1:
3783+
version "0.5.6"
3784+
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6"
3785+
integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==
3786+
dependencies:
3787+
minimist "^1.2.6"
3788+
37603789
mkdirp@^1.0.3:
37613790
version "1.0.4"
37623791
resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz"
@@ -4699,6 +4728,14 @@ source-map-support@^0.5.16, source-map-support@^0.5.17:
46994728
buffer-from "^1.0.0"
47004729
source-map "^0.6.0"
47014730

4731+
source-map-support@^0.5.6:
4732+
version "0.5.21"
4733+
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
4734+
integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==
4735+
dependencies:
4736+
buffer-from "^1.0.0"
4737+
source-map "^0.6.0"
4738+
47024739
source-map-url@^0.4.0:
47034740
version "0.4.0"
47044741
resolved "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz"
@@ -4875,6 +4912,11 @@ strip-ansi@^6.0.0:
48754912
dependencies:
48764913
ansi-regex "^5.0.0"
48774914

4915+
strip-bom@^3.0.0:
4916+
version "3.0.0"
4917+
resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
4918+
integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==
4919+
48784920
strip-bom@^4.0.0:
48794921
version "4.0.0"
48804922
resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz"
@@ -5065,6 +5107,29 @@ tr46@~0.0.3:
50655107
resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz"
50665108
integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
50675109

5110+
ts-mocha@^10.0.0:
5111+
version "10.0.0"
5112+
resolved "https://registry.yarnpkg.com/ts-mocha/-/ts-mocha-10.0.0.tgz#41a8d099ac90dbbc64b06976c5025ffaebc53cb9"
5113+
integrity sha512-VRfgDO+iiuJFlNB18tzOfypJ21xn2xbuZyDvJvqpTbWgkAgD17ONGr8t+Tl8rcBtOBdjXp5e/Rk+d39f7XBHRw==
5114+
dependencies:
5115+
ts-node "7.0.1"
5116+
optionalDependencies:
5117+
tsconfig-paths "^3.5.0"
5118+
5119+
[email protected]:
5120+
version "7.0.1"
5121+
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-7.0.1.tgz#9562dc2d1e6d248d24bc55f773e3f614337d9baf"
5122+
integrity sha512-BVwVbPJRspzNh2yfslyT1PSbl5uIk03EZlb493RKHN4qej/D06n1cEhjlOJG69oFsE7OT8XjpTUcYf6pKTLMhw==
5123+
dependencies:
5124+
arrify "^1.0.0"
5125+
buffer-from "^1.1.0"
5126+
diff "^3.1.0"
5127+
make-error "^1.1.1"
5128+
minimist "^1.2.0"
5129+
mkdirp "^0.5.1"
5130+
source-map-support "^0.5.6"
5131+
yn "^2.0.0"
5132+
50685133
ts-node@^10.0.0:
50695134
version "10.0.0"
50705135
resolved "https://registry.npmjs.org/ts-node/-/ts-node-10.0.0.tgz"
@@ -5081,6 +5146,16 @@ ts-node@^10.0.0:
50815146
source-map-support "^0.5.17"
50825147
yn "3.1.1"
50835148

5149+
tsconfig-paths@^3.5.0:
5150+
version "3.14.1"
5151+
resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a"
5152+
integrity sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==
5153+
dependencies:
5154+
"@types/json5" "^0.0.29"
5155+
json5 "^1.0.1"
5156+
minimist "^1.2.6"
5157+
strip-bom "^3.0.0"
5158+
50845159
tunnel-agent@^0.6.0:
50855160
version "0.6.0"
50865161
resolved "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz"
@@ -5563,3 +5638,8 @@ [email protected]:
55635638
version "3.1.1"
55645639
resolved "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz"
55655640
integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==
5641+
5642+
yn@^2.0.0:
5643+
version "2.0.0"
5644+
resolved "https://registry.yarnpkg.com/yn/-/yn-2.0.0.tgz#e5adabc8acf408f6385fc76495684c88e6af689a"
5645+
integrity sha512-uTv8J/wiWTgUTg+9vLTi//leUl5vDQS6uii/emeTb2ssY7vl6QWf2fFbIIGjnhjvbdKlU0ed7QPgY1htTC86jQ==

0 commit comments

Comments
 (0)