From bb19d8212a977298d23cb7f7206e3972a64407fe Mon Sep 17 00:00:00 2001 From: Anas Aboureada Date: Mon, 4 Feb 2019 17:18:39 +0100 Subject: [PATCH] fs: add fs.writev() which exposes syscall writev() fs with writev allow many buffers to be pushed to underlying OS APIs in one batch, so this should improve write speed to files. Refs: https://github.com/nodejs/node/issues/2298 PR-URL: https://github.com/nodejs/node/pull/25925 Fixes: https://github.com/nodejs/node/issues/2298 Reviewed-By: Anna Henningsen Reviewed-By: James M Snell Reviewed-By: Rich Trott --- doc/api/fs.md | 47 ++++++++++++++ lib/fs.js | 65 +++++++++++++++++++- test/parallel/test-fs-writev-sync.js | 87 ++++++++++++++++++++++++++ test/parallel/test-fs-writev.js | 92 ++++++++++++++++++++++++++++ tools/doc/type-parser.js | 8 +-- 5 files changed, 294 insertions(+), 5 deletions(-) create mode 100644 test/parallel/test-fs-writev-sync.js create mode 100644 test/parallel/test-fs-writev.js diff --git a/doc/api/fs.md b/doc/api/fs.md index d74570af037542..fbc6ca03046f88 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -3801,6 +3801,52 @@ changes: For detailed information, see the documentation of the asynchronous version of this API: [`fs.write(fd, string...)`][]. +## fs.writev(fd, buffers[, position], callback) + + +* `fd` {integer} +* `buffers` {ArrayBufferView[]} +* `position` {integer} +* `callback` {Function} + * `err` {Error} + * `bytesWritten` {integer} + * `buffers` {ArrayBufferView[]} + +Write an array of `ArrayBufferView`s to the file specified by `fd` using +`writev()`. + +`position` is the offset from the beginning of the file where this data +should be written. If `typeof position !== 'number'`, the data will be written +at the current position. + +The callback will be given three arguments: `err`, `bytesWritten`, and +`buffers`. `bytesWritten` is how many bytes were written from `buffers`. + +If this method is [`util.promisify()`][]ed, it returns a `Promise` for an +`Object` with `bytesWritten` and `buffers` properties. + +It is unsafe to use `fs.writev()` multiple times on the same file without +waiting for the callback. For this scenario, use [`fs.createWriteStream()`][]. + +On Linux, positional writes don't work when the file is opened in append mode. +The kernel ignores the position argument and always appends the data to +the end of the file. + +## fs.writevSync(fd, buffers[, position]) + + +* `fd` {integer} +* `buffers` {ArrayBufferView[]} +* `position` {integer} +* Returns: {number} The number of bytes written. + +For detailed information, see the documentation of the asynchronous version of +this API: [`fs.writev()`][]. + ## fs Promises API The `fs.promises` API provides an alternative set of asynchronous file system @@ -5051,6 +5097,7 @@ the file contents. [`fs.write(fd, buffer...)`]: #fs_fs_write_fd_buffer_offset_length_position_callback [`fs.write(fd, string...)`]: #fs_fs_write_fd_string_position_encoding_callback [`fs.writeFile()`]: #fs_fs_writefile_file_data_options_callback +[`fs.writev()`]: #fs_fs_writev_fd_buffers_position_callback [`inotify(7)`]: http://man7.org/linux/man-pages/man7/inotify.7.html [`kqueue(2)`]: https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 [`net.Socket`]: net.html#net_class_net_socket diff --git a/lib/fs.js b/lib/fs.js index 2ca3bdf21acab2..7b6b6d58ae31ef 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -142,6 +142,19 @@ function maybeCallback(cb) { throw new ERR_INVALID_CALLBACK(cb); } +function isBuffersArray(value) { + if (!Array.isArray(value)) + return false; + + for (var i = 0; i < value.length; i += 1) { + if (!isArrayBufferView(value[i])) { + return false; + } + } + + return true; +} + // Ensure that callbacks run in the global context. Only use this function // for callbacks that are passed to the binding layer, callbacks that are // invoked from JS already run in the proper scope. @@ -559,7 +572,7 @@ function write(fd, buffer, offset, length, position, callback) { Object.defineProperty(write, internalUtil.customPromisifyArgs, { value: ['bytesWritten', 'buffer'], enumerable: false }); -// usage: +// Usage: // fs.writeSync(fd, buffer[, offset[, length[, position]]]); // OR // fs.writeSync(fd, string[, position[, encoding]]); @@ -589,6 +602,54 @@ function writeSync(fd, buffer, offset, length, position) { return result; } +// usage: +// fs.writev(fd, buffers[, position], callback); +function writev(fd, buffers, position, callback) { + function wrapper(err, written) { + callback(err, written || 0, buffers); + } + + validateUint32(fd, 'fd'); + + if (!isBuffersArray(buffers)) { + throw new ERR_INVALID_ARG_TYPE('buffers', 'ArrayBufferView[]', buffers); + } + + const req = new FSReqCallback(); + req.oncomplete = wrapper; + + callback = maybeCallback(callback || position); + + if (typeof position !== 'number') + position = null; + + return binding.writeBuffers(fd, buffers, position, req); +} + +Object.defineProperty(writev, internalUtil.customPromisifyArgs, { + value: ['bytesWritten', 'buffer'], + enumerable: false +}); + +// fs.writevSync(fd, buffers[, position]); +function writevSync(fd, buffers, position) { + + validateUint32(fd, 'fd'); + const ctx = {}; + + if (!isBuffersArray(buffers)) { + throw new ERR_INVALID_ARG_TYPE('buffers', 'ArrayBufferView[]', buffers); + } + + if (typeof position !== 'number') + position = null; + + const result = binding.writeBuffers(fd, buffers, position, undefined, ctx); + + handleErrorFromBinding(ctx); + return result; +} + function rename(oldPath, newPath, callback) { callback = makeCallback(callback); oldPath = getValidatedPath(oldPath, 'oldPath'); @@ -1825,6 +1886,8 @@ module.exports = fs = { writeFileSync, write, writeSync, + writev, + writevSync, Dirent, Stats, diff --git a/test/parallel/test-fs-writev-sync.js b/test/parallel/test-fs-writev-sync.js new file mode 100644 index 00000000000000..a60115113b1f43 --- /dev/null +++ b/test/parallel/test-fs-writev-sync.js @@ -0,0 +1,87 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); +const tmpdir = require('../common/tmpdir'); + +tmpdir.refresh(); + +const expected = 'ümlaut. Лорем 運務ホソモ指及 आपको करने विकास 紙読決多密所 أضف'; + +const getFileName = (i) => path.join(tmpdir.path, `writev_sync_${i}.txt`); + +/** + * Testing with a array of buffers input + */ + +// fs.writevSync with array of buffers with all parameters +{ + const filename = getFileName(1); + const fd = fs.openSync(filename, 'w'); + + const buffer = Buffer.from(expected); + const bufferArr = [buffer, buffer]; + const expectedLength = bufferArr.length * buffer.byteLength; + + let written = fs.writevSync(fd, [Buffer.from('')], null); + assert.deepStrictEqual(written, 0); + + written = fs.writevSync(fd, bufferArr, null); + assert.deepStrictEqual(written, expectedLength); + + fs.closeSync(fd); + + assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename))); +} + +// fs.writevSync with array of buffers without position +{ + const filename = getFileName(2); + const fd = fs.openSync(filename, 'w'); + + const buffer = Buffer.from(expected); + const bufferArr = [buffer, buffer, buffer]; + const expectedLength = bufferArr.length * buffer.byteLength; + + let written = fs.writevSync(fd, [Buffer.from('')]); + assert.deepStrictEqual(written, 0); + + written = fs.writevSync(fd, bufferArr); + assert.deepStrictEqual(written, expectedLength); + + fs.closeSync(fd); + + assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename))); +} + +/** + * Testing with wrong input types + */ +{ + const filename = getFileName(3); + const fd = fs.openSync(filename, 'w'); + + [false, 'test', {}, [{}], ['sdf'], null, undefined].forEach((i) => { + common.expectsError( + () => fs.writevSync(fd, i, null), { + code: 'ERR_INVALID_ARG_TYPE', + type: TypeError + } + ); + }); + + fs.closeSync(fd); +} + +// fs.writevSync with wrong fd types +[false, 'test', {}, [{}], null, undefined].forEach((i) => { + common.expectsError( + () => fs.writevSync(i), + { + code: 'ERR_INVALID_ARG_TYPE', + type: TypeError + } + ); +}); diff --git a/test/parallel/test-fs-writev.js b/test/parallel/test-fs-writev.js new file mode 100644 index 00000000000000..6a66c631f99712 --- /dev/null +++ b/test/parallel/test-fs-writev.js @@ -0,0 +1,92 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); +const tmpdir = require('../common/tmpdir'); + +tmpdir.refresh(); + +const expected = 'ümlaut. Лорем 運務ホソモ指及 आपको करने विकास 紙読決多密所 أضف'; + +const getFileName = (i) => path.join(tmpdir.path, `writev_${i}.txt`); + +/** + * Testing with a array of buffers input + */ + +// fs.writev with array of buffers with all parameters +{ + const filename = getFileName(1); + const fd = fs.openSync(filename, 'w'); + + const buffer = Buffer.from(expected); + const bufferArr = [buffer, buffer]; + + const done = common.mustCall((err, written, buffers) => { + assert.ifError(err); + + assert.deepStrictEqual(bufferArr, buffers); + const expectedLength = bufferArr.length * buffer.byteLength; + assert.deepStrictEqual(written, expectedLength); + fs.closeSync(fd); + + assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename))); + }); + + fs.writev(fd, bufferArr, null, done); +} + +// fs.writev with array of buffers without position +{ + const filename = getFileName(2); + const fd = fs.openSync(filename, 'w'); + + const buffer = Buffer.from(expected); + const bufferArr = [buffer, buffer]; + + const done = common.mustCall((err, written, buffers) => { + assert.ifError(err); + + assert.deepStrictEqual(bufferArr, buffers); + + const expectedLength = bufferArr.length * buffer.byteLength; + assert.deepStrictEqual(written, expectedLength); + fs.closeSync(fd); + + assert(Buffer.concat(bufferArr).equals(fs.readFileSync(filename))); + }); + + fs.writev(fd, bufferArr, done); +} + +/** + * Testing with wrong input types + */ +{ + const filename = getFileName(3); + const fd = fs.openSync(filename, 'w'); + + [false, 'test', {}, [{}], ['sdf'], null, undefined].forEach((i) => { + common.expectsError( + () => fs.writev(fd, i, null, common.mustNotCall()), { + code: 'ERR_INVALID_ARG_TYPE', + type: TypeError + } + ); + }); + + fs.closeSync(fd); +} + +// fs.writev with wrong fd types +[false, 'test', {}, [{}], null, undefined].forEach((i) => { + common.expectsError( + () => fs.writev(i, common.mustNotCall()), + { + code: 'ERR_INVALID_ARG_TYPE', + type: TypeError + } + ); +}); diff --git a/tools/doc/type-parser.js b/tools/doc/type-parser.js index 22307bdf6e4d3c..6acc4196a36d99 100644 --- a/tools/doc/type-parser.js +++ b/tools/doc/type-parser.js @@ -15,10 +15,10 @@ const jsPrimitives = { const jsGlobalObjectsUrl = `${jsDocPrefix}Reference/Global_Objects/`; const jsGlobalTypes = [ - 'Array', 'ArrayBuffer', 'DataView', 'Date', 'Error', 'EvalError', 'Function', - 'Map', 'Object', 'Promise', 'RangeError', 'ReferenceError', 'RegExp', 'Set', - 'SharedArrayBuffer', 'SyntaxError', 'TypeError', 'TypedArray', 'URIError', - 'Uint8Array', + 'Array', 'ArrayBuffer', 'ArrayBufferView', 'DataView', 'Date', 'Error', + 'EvalError', 'Function', 'Map', 'Object', 'Promise', 'RangeError', + 'ReferenceError', 'RegExp', 'Set', 'SharedArrayBuffer', 'SyntaxError', + 'TypeError', 'TypedArray', 'URIError', 'Uint8Array', ]; const customTypesMap = {