-
-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
6 changed files
with
76 additions
and
106 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,7 +5,7 @@ | |
Break up a stream and reassemble it so that each line is a chunk. | ||
`split2` is inspired by [@dominictarr](https://github.com/dominictarr) [`split`](https://github.com/dominictarr/split) module, | ||
and it is totally API compatible with it. | ||
However, it is based on Node.js core [`Transform`](https://nodejs.org/api/stream.html#stream_new_stream_transform_options) via [`readable-stream`](https://github.com/nodejs/readable-stream) | ||
However, it is based on Node.js core [`Transform`](https://nodejs.org/api/stream.html#stream_new_stream_transform_options). | ||
|
||
`matcher` may be a `String`, or a `RegExp`. Example, read every line in a file ... | ||
|
||
|
@@ -68,21 +68,9 @@ fs.createReadStream(file) | |
However, in [@dominictarr](https://github.com/dominictarr) [`split`](https://github.com/dominictarr/split) the mapper | ||
is wrapped in a try-catch, while here it is not: if your parsing logic can throw, wrap it yourself. Otherwise, you can also use the stream error handling when mapper function throw. | ||
|
||
# Benchmark | ||
|
||
```bash | ||
$ node bench.js | ||
benchSplit*10000: 1484.983ms | ||
benchBinarySplit*10000: 1484.080ms | ||
benchSplit*10000: 1407.334ms | ||
benchBinarySplit*10000: 1500.281ms | ||
``` | ||
|
||
Benchmark taken on Node 8.11.3, on a Macbook i5 2018. | ||
|
||
# License | ||
|
||
Copyright (c) 2014-2018, Matteo Collina <[email protected]> | ||
Copyright (c) 2014-2021, Matteo Collina <[email protected]> | ||
|
||
Permission to use, copy, modify, and/or distribute this software for any | ||
purpose with or without fee is hereby granted, provided that the above | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,5 @@ | ||
/* | ||
Copyright (c) 2014-2018, Matteo Collina <[email protected]> | ||
Copyright (c) 2014-2021, Matteo Collina <[email protected]> | ||
Permission to use, copy, modify, and/or distribute this software for any | ||
purpose with or without fee is hereby granted, provided that the above | ||
|
@@ -16,15 +16,15 @@ IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | |
|
||
'use strict' | ||
|
||
const { Transform } = require('readable-stream') | ||
const { Transform } = require('stream') | ||
const { StringDecoder } = require('string_decoder') | ||
const kLast = Symbol('last') | ||
const kDecoder = Symbol('decoder') | ||
|
||
function transform (chunk, enc, cb) { | ||
var list | ||
let list | ||
if (this.overflow) { // Line buffer is full. Skip to start of next line. | ||
var buf = this[kDecoder].write(chunk) | ||
const buf = this[kDecoder].write(chunk) | ||
list = buf.split(this.matcher) | ||
|
||
if (list.length === 1) return cb() // Line ending not found. Discard entire chunk. | ||
|
@@ -39,7 +39,7 @@ function transform (chunk, enc, cb) { | |
|
||
this[kLast] = list.pop() | ||
|
||
for (var i = 0; i < list.length; i++) { | ||
for (let i = 0; i < list.length; i++) { | ||
try { | ||
push(this, this.mapper(list[i])) | ||
} catch (error) { | ||
|
@@ -48,7 +48,10 @@ function transform (chunk, enc, cb) { | |
} | ||
|
||
this.overflow = this[kLast].length > this.maxLength | ||
if (this.overflow && !this.skipOverflow) return cb(new Error('maximum buffer reached')) | ||
if (this.overflow && !this.skipOverflow) { | ||
cb(new Error('maximum buffer reached')) | ||
return | ||
} | ||
|
||
cb() | ||
} | ||
|
@@ -112,6 +115,7 @@ function split (matcher, mapper, options) { | |
} | ||
|
||
options = Object.assign({}, options) | ||
options.autoDestroy = true | ||
options.transform = transform | ||
options.flush = flush | ||
options.readableObjectMode = true | ||
|
@@ -123,8 +127,13 @@ function split (matcher, mapper, options) { | |
stream.matcher = matcher | ||
stream.mapper = mapper | ||
stream.maxLength = options.maxLength | ||
stream.skipOverflow = options.skipOverflow | ||
stream.skipOverflow = options.skipOverflow || false | ||
stream.overflow = false | ||
stream._destroy = function (err, cb) { | ||
// Weird Node v12 bug that we need to work around | ||
this._writableState.errorEmitted = false | ||
cb(err) | ||
} | ||
|
||
return stream | ||
} | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.