Skip to content

Commit

Permalink
fs: add fs.writev() which exposes syscall writev()
Browse files Browse the repository at this point in the history
fs with writev allow many buffers to be pushed to underlying OS
APIs in one batch, so this should improve write speed to files.

Refs: #2298

PR-URL: #25925
Fixes: #2298
Reviewed-By: Anna Henningsen <[email protected]>
Reviewed-By: James M Snell <[email protected]>
Reviewed-By: Rich Trott <[email protected]>
  • Loading branch information
AnasAboreeda authored and targos committed Aug 19, 2019
1 parent 5b892c4 commit bb19d82
Show file tree
Hide file tree
Showing 5 changed files with 294 additions and 5 deletions.
47 changes: 47 additions & 0 deletions doc/api/fs.md
Original file line number Diff line number Diff line change
Expand Up @@ -3801,6 +3801,52 @@ changes:
For detailed information, see the documentation of the asynchronous version of
this API: [`fs.write(fd, string...)`][].

## fs.writev(fd, buffers[, position], callback)
<!-- YAML
added: REPLACEME
-->

* `fd` {integer}
* `buffers` {ArrayBufferView[]}
* `position` {integer}
* `callback` {Function}
* `err` {Error}
* `bytesWritten` {integer}
* `buffers` {ArrayBufferView[]}

Write an array of `ArrayBufferView`s to the file specified by `fd` using
`writev()`.

`position` is the offset from the beginning of the file where this data
should be written. If `typeof position !== 'number'`, the data will be written
at the current position.

The callback will be given three arguments: `err`, `bytesWritten`, and
`buffers`. `bytesWritten` is how many bytes were written from `buffers`.

If this method is [`util.promisify()`][]ed, it returns a `Promise` for an
`Object` with `bytesWritten` and `buffers` properties.

It is unsafe to use `fs.writev()` multiple times on the same file without
waiting for the callback. For this scenario, use [`fs.createWriteStream()`][].

On Linux, positional writes don't work when the file is opened in append mode.
The kernel ignores the position argument and always appends the data to
the end of the file.

## fs.writevSync(fd, buffers[, position])
<!-- YAML
added: REPLACEME
-->

* `fd` {integer}
* `buffers` {ArrayBufferView[]}
* `position` {integer}
* Returns: {number} The number of bytes written.

For detailed information, see the documentation of the asynchronous version of
this API: [`fs.writev()`][].

## fs Promises API

The `fs.promises` API provides an alternative set of asynchronous file system
Expand Down Expand Up @@ -5051,6 +5097,7 @@ the file contents.
[`fs.write(fd, buffer...)`]: #fs_fs_write_fd_buffer_offset_length_position_callback
[`fs.write(fd, string...)`]: #fs_fs_write_fd_string_position_encoding_callback
[`fs.writeFile()`]: #fs_fs_writefile_file_data_options_callback
[`fs.writev()`]: #fs_fs_writev_fd_buffers_position_callback
[`inotify(7)`]: http://man7.org/linux/man-pages/man7/inotify.7.html
[`kqueue(2)`]: https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2
[`net.Socket`]: net.html#net_class_net_socket
Expand Down
65 changes: 64 additions & 1 deletion lib/fs.js
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,19 @@ function maybeCallback(cb) {
throw new ERR_INVALID_CALLBACK(cb);
}

function isBuffersArray(value) {
if (!Array.isArray(value))
return false;

for (var i = 0; i < value.length; i += 1) {
if (!isArrayBufferView(value[i])) {
return false;
}
}

return true;
}

// Ensure that callbacks run in the global context. Only use this function
// for callbacks that are passed to the binding layer, callbacks that are
// invoked from JS already run in the proper scope.
Expand Down Expand Up @@ -559,7 +572,7 @@ function write(fd, buffer, offset, length, position, callback) {
Object.defineProperty(write, internalUtil.customPromisifyArgs,
{ value: ['bytesWritten', 'buffer'], enumerable: false });

// usage:
// Usage:
// fs.writeSync(fd, buffer[, offset[, length[, position]]]);
// OR
// fs.writeSync(fd, string[, position[, encoding]]);
Expand Down Expand Up @@ -589,6 +602,54 @@ function writeSync(fd, buffer, offset, length, position) {
return result;
}

// usage:
// fs.writev(fd, buffers[, position], callback);
function writev(fd, buffers, position, callback) {
function wrapper(err, written) {
callback(err, written || 0, buffers);
}

validateUint32(fd, 'fd');

if (!isBuffersArray(buffers)) {
throw new ERR_INVALID_ARG_TYPE('buffers', 'ArrayBufferView[]', buffers);
}

const req = new FSReqCallback();
req.oncomplete = wrapper;

callback = maybeCallback(callback || position);

if (typeof position !== 'number')
position = null;

return binding.writeBuffers(fd, buffers, position, req);
}

Object.defineProperty(writev, internalUtil.customPromisifyArgs, {
value: ['bytesWritten', 'buffer'],
enumerable: false
});

// fs.writevSync(fd, buffers[, position]);
function writevSync(fd, buffers, position) {

validateUint32(fd, 'fd');
const ctx = {};

if (!isBuffersArray(buffers)) {
throw new ERR_INVALID_ARG_TYPE('buffers', 'ArrayBufferView[]', buffers);
}

if (typeof position !== 'number')
position = null;

const result = binding.writeBuffers(fd, buffers, position, undefined, ctx);

handleErrorFromBinding(ctx);
return result;
}

function rename(oldPath, newPath, callback) {
callback = makeCallback(callback);
oldPath = getValidatedPath(oldPath, 'oldPath');
Expand Down Expand Up @@ -1825,6 +1886,8 @@ module.exports = fs = {
writeFileSync,
write,
writeSync,
writev,
writevSync,
Dirent,
Stats,

Expand Down
87 changes: 87 additions & 0 deletions test/parallel/test-fs-writev-sync.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
'use strict';

const common = require('../common');
const assert = require('assert');
const path = require('path');
const fs = require('fs');
const tmpdir = require('../common/tmpdir');

tmpdir.refresh();

const expected = 'ümlaut. Лорем 運務ホソモ指及 आपको करने विकास 紙読決多密所 أضف';

const getFileName = (i) => path.join(tmpdir.path, `writev_sync_${i}.txt`);

/**
* Testing with a array of buffers input
*/

// fs.writevSync with array of buffers with all parameters
{
const filename = getFileName(1);
const fd = fs.openSync(filename, 'w');

const buffer = Buffer.from(expected);
const bufferArr = [buffer, buffer];
const expectedLength = bufferArr.length * buffer.byteLength;

let written = fs.writevSync(fd, [Buffer.from('')], null);
assert.deepStrictEqual(written, 0);

written = fs.writevSync(fd, bufferArr, null);
assert.deepStrictEqual(written, expectedLength);

fs.closeSync(fd);

assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename)));
}

// fs.writevSync with array of buffers without position
{
const filename = getFileName(2);
const fd = fs.openSync(filename, 'w');

const buffer = Buffer.from(expected);
const bufferArr = [buffer, buffer, buffer];
const expectedLength = bufferArr.length * buffer.byteLength;

let written = fs.writevSync(fd, [Buffer.from('')]);
assert.deepStrictEqual(written, 0);

written = fs.writevSync(fd, bufferArr);
assert.deepStrictEqual(written, expectedLength);

fs.closeSync(fd);

assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename)));
}

/**
* Testing with wrong input types
*/
{
const filename = getFileName(3);
const fd = fs.openSync(filename, 'w');

[false, 'test', {}, [{}], ['sdf'], null, undefined].forEach((i) => {
common.expectsError(
() => fs.writevSync(fd, i, null), {
code: 'ERR_INVALID_ARG_TYPE',
type: TypeError
}
);
});

fs.closeSync(fd);
}

// fs.writevSync with wrong fd types
[false, 'test', {}, [{}], null, undefined].forEach((i) => {
common.expectsError(
() => fs.writevSync(i),
{
code: 'ERR_INVALID_ARG_TYPE',
type: TypeError
}
);
});
92 changes: 92 additions & 0 deletions test/parallel/test-fs-writev.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
'use strict';

const common = require('../common');
const assert = require('assert');
const path = require('path');
const fs = require('fs');
const tmpdir = require('../common/tmpdir');

tmpdir.refresh();

const expected = 'ümlaut. Лорем 運務ホソモ指及 आपको करने विकास 紙読決多密所 أضف';

const getFileName = (i) => path.join(tmpdir.path, `writev_${i}.txt`);

/**
* Testing with a array of buffers input
*/

// fs.writev with array of buffers with all parameters
{
const filename = getFileName(1);
const fd = fs.openSync(filename, 'w');

const buffer = Buffer.from(expected);
const bufferArr = [buffer, buffer];

const done = common.mustCall((err, written, buffers) => {
assert.ifError(err);

assert.deepStrictEqual(bufferArr, buffers);
const expectedLength = bufferArr.length * buffer.byteLength;
assert.deepStrictEqual(written, expectedLength);
fs.closeSync(fd);

assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename)));
});

fs.writev(fd, bufferArr, null, done);
}

// fs.writev with array of buffers without position
{
const filename = getFileName(2);
const fd = fs.openSync(filename, 'w');

const buffer = Buffer.from(expected);
const bufferArr = [buffer, buffer];

const done = common.mustCall((err, written, buffers) => {
assert.ifError(err);

assert.deepStrictEqual(bufferArr, buffers);

const expectedLength = bufferArr.length * buffer.byteLength;
assert.deepStrictEqual(written, expectedLength);
fs.closeSync(fd);

assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename)));
});

fs.writev(fd, bufferArr, done);
}

/**
* Testing with wrong input types
*/
{
const filename = getFileName(3);
const fd = fs.openSync(filename, 'w');

[false, 'test', {}, [{}], ['sdf'], null, undefined].forEach((i) => {
common.expectsError(
() => fs.writev(fd, i, null, common.mustNotCall()), {
code: 'ERR_INVALID_ARG_TYPE',
type: TypeError
}
);
});

fs.closeSync(fd);
}

// fs.writev with wrong fd types
[false, 'test', {}, [{}], null, undefined].forEach((i) => {
common.expectsError(
() => fs.writev(i, common.mustNotCall()),
{
code: 'ERR_INVALID_ARG_TYPE',
type: TypeError
}
);
});
8 changes: 4 additions & 4 deletions tools/doc/type-parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ const jsPrimitives = {

const jsGlobalObjectsUrl = `${jsDocPrefix}Reference/Global_Objects/`;
const jsGlobalTypes = [
'Array', 'ArrayBuffer', 'DataView', 'Date', 'Error', 'EvalError', 'Function',
'Map', 'Object', 'Promise', 'RangeError', 'ReferenceError', 'RegExp', 'Set',
'SharedArrayBuffer', 'SyntaxError', 'TypeError', 'TypedArray', 'URIError',
'Uint8Array',
'Array', 'ArrayBuffer', 'ArrayBufferView', 'DataView', 'Date', 'Error',
'EvalError', 'Function', 'Map', 'Object', 'Promise', 'RangeError',
'ReferenceError', 'RegExp', 'Set', 'SharedArrayBuffer', 'SyntaxError',
'TypeError', 'TypedArray', 'URIError', 'Uint8Array',
];

const customTypesMap = {
Expand Down

0 comments on commit bb19d82

Please sign in to comment.