Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stream: check type and range of highWaterMark #18098

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 3 additions & 13 deletions lib/_stream_readable.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const util = require('util');
const debug = util.debuglog('stream');
const BufferList = require('internal/streams/BufferList');
const destroyImpl = require('internal/streams/destroy');
const { getHighWaterMark } = require('internal/streams/state');
const errors = require('internal/errors');
var StringDecoder;

Expand Down Expand Up @@ -75,19 +76,8 @@ function ReadableState(options, stream) {

// the point at which it stops calling _read() to fill the buffer
// Note: 0 is a valid value, means "don't call _read preemptively ever"
var hwm = options.highWaterMark;
var readableHwm = options.readableHighWaterMark;
var defaultHwm = this.objectMode ? 16 : 16 * 1024;

if (hwm || hwm === 0)
this.highWaterMark = hwm;
else if (isDuplex && (readableHwm || readableHwm === 0))
this.highWaterMark = readableHwm;
else
this.highWaterMark = defaultHwm;

// cast to ints.
this.highWaterMark = Math.floor(this.highWaterMark);
this.highWaterMark = getHighWaterMark(this, options, 'readableHighWaterMark',
isDuplex);

// A linked list is used to store data chunks instead of an array because the
// linked list can remove elements from the beginning faster than
Expand Down
16 changes: 3 additions & 13 deletions lib/_stream_writable.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const internalUtil = require('internal/util');
const Stream = require('stream');
const { Buffer } = require('buffer');
const destroyImpl = require('internal/streams/destroy');
const { getHighWaterMark } = require('internal/streams/state');
const errors = require('internal/errors');

util.inherits(Writable, Stream);
Expand All @@ -59,19 +60,8 @@ function WritableState(options, stream) {
// the point at which write() starts returning false
// Note: 0 is a valid value, means that we always return false if
// the entire buffer is not flushed immediately on write()
var hwm = options.highWaterMark;
var writableHwm = options.writableHighWaterMark;
var defaultHwm = this.objectMode ? 16 : 16 * 1024;

if (hwm || hwm === 0)
this.highWaterMark = hwm;
else if (isDuplex && (writableHwm || writableHwm === 0))
this.highWaterMark = writableHwm;
else
this.highWaterMark = defaultHwm;

// cast to ints.
this.highWaterMark = Math.floor(this.highWaterMark);
this.highWaterMark = getHighWaterMark(this, options, 'writableHighWaterMark',
isDuplex);

// if _final has been called
this.finalCalled = false;
Expand Down
26 changes: 26 additions & 0 deletions lib/internal/streams/state.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
'use strict';

const errors = require('internal/errors');

function getHighWaterMark(state, options, duplexKey, isDuplex) {
let hwm = options.highWaterMark;
if (hwm != null) {
if (typeof hwm !== 'number' || !(hwm >= 0))
throw new errors.TypeError('ERR_INVALID_OPT_VALUE', 'highWaterMark', hwm);
return Math.floor(hwm);
} else if (isDuplex) {
hwm = options[duplexKey];
if (hwm != null) {
if (typeof hwm !== 'number' || !(hwm >= 0))
throw new errors.TypeError('ERR_INVALID_OPT_VALUE', duplexKey, hwm);
return Math.floor(hwm);
}
}

// Default value
return state.objectMode ? 16 : 16 * 1024;
}
Copy link
Member

@BridgeAR BridgeAR Jan 11, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The following should be more efficient (untested):

function getHighWaterMark (options, isDuplex, specialKey, objectMode) {
  if (typeof options.highWaterMark === 'number') {
    if (options.highWaterMark < 0 || Number.isNaN(options.highWaterMark)) {
      throw new errors.TypeError(
        'ERR_INVALID_OPT_VALUE',
        'options.highWaterMark',
        options.highWaterMark
      )
    }
    return Math.floor(options.highWaterMark)
  }
  if (isDuplex) {
    if (typeof options[specialKey] === 'number') {
      if (options[specialKey] < 0 || Number.isNaN(options[specialKey])) {
        throw new TypeError(...)
      }
      return Math.floor(options[specialKey])
    }
    if (options[specialKey]) {
      throw new TypeError(...)
    }
  } else if(options.highWaterMark) {
    throw new TypeError(...)
  }

  return objectMode ? 16 : 16 * 1024
}

getHighWaterMark(
  options,
  isDuplex,
  'writableHighWaterMark', // Replace accordingly
  this.objectMode
)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I was going to extract the key anyway, just wanted to hear opinions about NaN first (that really annoyed me).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By the way, !(options.highWaterMark >= 0) should be faster than options.highWaterMark < 0 || Number.isNaN(options.highWaterMark).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is probably true.


module.exports = {
getHighWaterMark
};
1 change: 1 addition & 0 deletions node.gyp
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@
'lib/internal/streams/BufferList.js',
'lib/internal/streams/legacy.js',
'lib/internal/streams/destroy.js',
'lib/internal/streams/state.js',
'lib/internal/wrap_js_stream.js',
'deps/v8/tools/splaytree.js',
'deps/v8/tools/codemap.js',
Expand Down
25 changes: 22 additions & 3 deletions test/parallel/test-stream-transform-split-highwatermark.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
'use strict';
require('../common');
const common = require('../common');
const assert = require('assert');

const { Transform, Readable, Writable } = require('stream');
Expand Down Expand Up @@ -54,14 +54,33 @@ testTransform(0, 0, {
writableHighWaterMark: 777,
});

// test undefined, null, NaN
[undefined, null, NaN].forEach((v) => {
// test undefined, null
[undefined, null].forEach((v) => {
testTransform(DEFAULT, DEFAULT, { readableHighWaterMark: v });
testTransform(DEFAULT, DEFAULT, { writableHighWaterMark: v });
testTransform(666, DEFAULT, { highWaterMark: v, readableHighWaterMark: 666 });
testTransform(DEFAULT, 777, { highWaterMark: v, writableHighWaterMark: 777 });
});

// test NaN
{
common.expectsError(() => {
new Transform({ readableHighWaterMark: NaN });
}, {
type: TypeError,
code: 'ERR_INVALID_OPT_VALUE',
message: 'The value "NaN" is invalid for option "readableHighWaterMark"'
});

common.expectsError(() => {
new Transform({ writableHighWaterMark: NaN });
}, {
type: TypeError,
code: 'ERR_INVALID_OPT_VALUE',
message: 'The value "NaN" is invalid for option "writableHighWaterMark"'
});
}

// test non Duplex streams ignore the options
{
const r = new Readable({ readableHighWaterMark: 666 });
Expand Down
17 changes: 15 additions & 2 deletions test/parallel/test-streams-highwatermark.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
'use strict';
require('../common');
const common = require('../common');

// This test ensures that the stream implementation correctly handles values
// for highWaterMark which exceed the range of signed 32 bit integers.
// for highWaterMark which exceed the range of signed 32 bit integers and
// rejects invalid values.

const assert = require('assert');
const stream = require('stream');
Expand All @@ -16,3 +17,15 @@ assert.strictEqual(readable._readableState.highWaterMark, ovfl);

const writable = stream.Writable({ highWaterMark: ovfl });
assert.strictEqual(writable._writableState.highWaterMark, ovfl);

for (const invalidHwm of [true, false, '5', {}, -5, NaN]) {
for (const type of [stream.Readable, stream.Writable]) {
common.expectsError(() => {
type({ highWaterMark: invalidHwm });
}, {
type: TypeError,
code: 'ERR_INVALID_OPT_VALUE',
message: `The value "${invalidHwm}" is invalid for option "highWaterMark"`
});
}
}