diff --git a/graceful-fs.js b/graceful-fs.js index 8d5b89e..52d6428 100644 --- a/graceful-fs.js +++ b/graceful-fs.js @@ -93,6 +93,13 @@ if (!global[gracefulQueue]) { publishQueue(global, fs[gracefulQueue]); } +// Node's writeFile/writeAll call the exported fs.write with the full +// buffer length. fs.write rejects length > INT32_MAX (2GiB - 1), so +// graceful-fs writeFile fails for buffers >= 2GiB while fs.promises +// writeFile already chunks. Patch the real fs.write that writeAll uses. +// See https://github.com/isaacs/node-graceful-fs/issues/256 +patchLargeWrites() + module.exports = patch(clone(fs)) if (process.env.TEST_GRACEFUL_FS_GLOBAL_PATCH && !fs.__patched) { module.exports = patch(fs) @@ -446,3 +453,198 @@ function retry () { retryTimer = setTimeout(retry, 0) } } + +// uv_fs_write / fs.write take a signed 32-bit length. +var MAX_WRITE = 2147483647 + +function patchLargeWrites () { + if (typeof fs.write === 'function' && + fs.write.toString().indexOf('chunk writes larger than INT32_MAX') === -1) { + fs.write = wrapWrite(fs.write) + } + if (typeof fs.writeSync === 'function' && + fs.writeSync.toString().indexOf('chunk writes larger than INT32_MAX') === -1) { + fs.writeSync = wrapWriteSync(fs.writeSync) + } +} + +function isBinaryWriteTarget (buffer) { + return buffer != null && + typeof buffer !== 'string' && + typeof buffer.byteLength === 'number' +} + +function copyWriteSymbols (from, to) { + if (typeof Object.getOwnPropertySymbols !== 'function') + return + var symbols = Object.getOwnPropertySymbols(from) + for (var i = 0; i < symbols.length; i++) { + var desc = Object.getOwnPropertyDescriptor(from, symbols[i]) + if (desc) + Object.defineProperty(to, symbols[i], desc) + } +} + +function parseBinaryWrite (buffer, offset, length, position, callback) { + if (typeof offset === 'function') { + return { + offset: 0, + length: buffer.byteLength, + position: null, + cb: offset + } + } + if (typeof length === 'function') { + var off1 = typeof offset === 'number' ? offset : 0 + return { + offset: off1, + length: buffer.byteLength - off1, + position: null, + cb: length + } + } + var off = typeof offset === 'number' ? offset : 0 + var len = typeof length === 'number' ? length : buffer.byteLength - off + if (typeof position === 'function') { + return { + offset: off, + length: len, + position: null, + cb: position + } + } + return { + offset: off, + length: len, + position: typeof position === 'number' ? position : null, + cb: callback + } +} + +function writeLarge (writeFn, fd, buffer, offset, length, position, callback) { + var remaining = length + var bufOffset = offset + var filePos = position + var total = 0 + + function onWrite (er, written) { + if (er) { + if (typeof callback === 'function') + callback(er, total, buffer) + return + } + written = written || 0 + total += written + remaining -= written + bufOffset += written + if (typeof filePos === 'number') + filePos += written + if (remaining <= 0) { + if (typeof callback === 'function') + callback(null, total, buffer) + return + } + if (written === 0) { + if (typeof callback === 'function') + callback(null, total, buffer) + return + } + doWrite() + } + + function doWrite () { + var chunk = remaining > MAX_WRITE ? MAX_WRITE : remaining + writeFn.call(fs, fd, buffer, bufOffset, chunk, filePos, onWrite) + } + + doWrite() +} + +function writeLargeSync (writeFn, fd, buffer, offset, length, position) { + var remaining = length + var bufOffset = offset + var filePos = position + var total = 0 + while (remaining > 0) { + var chunk = remaining > MAX_WRITE ? MAX_WRITE : remaining + var written = writeFn.call(fs, fd, buffer, bufOffset, chunk, filePos) + written = written || 0 + total += written + remaining -= written + bufOffset += written + if (typeof filePos === 'number') + filePos += written + if (written === 0) + break + } + return total +} + +function wrapWrite (fs$write) { + function write (fd, buffer, offset, length, position, callback) { + // graceful-fs: chunk writes larger than INT32_MAX + if (isBinaryWriteTarget(buffer)) { + if (offset !== null && typeof offset === 'object') { + var cbOpt = length + if (typeof cbOpt === 'function') { + var ooff = typeof offset.offset === 'number' ? offset.offset : 0 + var olen = typeof offset.length === 'number' ? offset.length : buffer.byteLength - ooff + var opos = typeof offset.position === 'number' ? offset.position : null + if (olen > MAX_WRITE) { + return writeLarge(fs$write, fd, buffer, ooff, olen, opos, cbOpt) + } + } + } else { + var parsed = parseBinaryWrite(buffer, offset, length, position, callback) + if (parsed.length > MAX_WRITE && typeof parsed.cb === 'function') { + return writeLarge(fs$write, fd, buffer, parsed.offset, parsed.length, parsed.position, parsed.cb) + } + } + } + return fs$write.apply(this, arguments) + } + + if (Object.setPrototypeOf) + Object.setPrototypeOf(write, fs$write) + copyWriteSymbols(fs$write, write) + if (previousSymbol) { + Object.defineProperty(write, previousSymbol, { + value: fs$write + }) + } + return write +} + +function wrapWriteSync (fs$writeSync) { + function writeSync (fd, buffer, offset, length, position) { + // graceful-fs: chunk writes larger than INT32_MAX + if (isBinaryWriteTarget(buffer)) { + if (offset !== null && typeof offset === 'object') { + var ooff = typeof offset.offset === 'number' ? offset.offset : 0 + var olen = typeof offset.length === 'number' ? offset.length : buffer.byteLength - ooff + var opos = typeof offset.position === 'number' ? offset.position : null + if (olen > MAX_WRITE) { + return writeLargeSync(fs$writeSync, fd, buffer, ooff, olen, opos) + } + } else { + var off = typeof offset === 'number' ? offset : 0 + var len = typeof length === 'number' ? length : buffer.byteLength - off + var pos = typeof position === 'number' ? position : null + if (len > MAX_WRITE) { + return writeLargeSync(fs$writeSync, fd, buffer, off, len, pos) + } + } + } + return fs$writeSync.apply(this, arguments) + } + + if (Object.setPrototypeOf) + Object.setPrototypeOf(writeSync, fs$writeSync) + copyWriteSymbols(fs$writeSync, writeSync) + if (previousSymbol) { + Object.defineProperty(writeSync, previousSymbol, { + value: fs$writeSync + }) + } + return writeSync +} diff --git a/test/write-large-buffer.js b/test/write-large-buffer.js new file mode 100644 index 0000000..863c2b1 --- /dev/null +++ b/test/write-large-buffer.js @@ -0,0 +1,158 @@ +'use strict' + +var fs = require('fs') +var path = require('path') +var t = require('tap') + +// Must wrap the exported fs.write that writeAll uses *before* loading +// graceful-fs, so the chunking wrapper captures this spy. +var INT32_MAX = 2147483647 +var writes = [] + +function fakeWrite (fd, buffer, offset, length, position, cb) { + if (typeof position === 'function') { + cb = position + position = null + } + if (typeof length === 'function') { + cb = length + length = buffer.byteLength - (typeof offset === 'number' ? offset : 0) + position = null + } + if (typeof offset === 'function') { + cb = offset + offset = 0 + length = buffer.byteLength + position = null + } + writes.push({ + offset: offset, + length: length, + position: position + }) + process.nextTick(function () { + cb(null, length, buffer) + }) +} + +function fakeWriteSync (fd, buffer, offset, length, position) { + if (typeof offset !== 'number') + offset = 0 + if (typeof length !== 'number') + length = buffer.byteLength - offset + writes.push({ + offset: offset, + length: length, + position: position, + sync: true + }) + return length +} + +fs.write = fakeWrite +fs.writeSync = fakeWriteSync + +var gfs = require('../') + +t.test('write() does not chunk lengths at or below INT32_MAX', function (t) { + writes.length = 0 + var buf = Buffer.alloc(16) + gfs.write(1, buf, 0, 16, null, function (err, written, buffer) { + t.error(err) + t.equal(written, 16) + t.equal(buffer, buf) + t.equal(writes.length, 1) + t.equal(writes[0].length, 16) + t.end() + }) +}) + +t.test('write() chunks length greater than INT32_MAX', function (t) { + writes.length = 0 + var buf = Buffer.alloc(8) + gfs.write(1, buf, 0, INT32_MAX + 1, null, function (err, written, buffer) { + t.error(err) + t.equal(written, INT32_MAX + 1) + t.equal(buffer, buf) + t.equal(writes.length, 2) + t.equal(writes[0].offset, 0) + t.equal(writes[0].length, INT32_MAX) + t.equal(writes[1].offset, INT32_MAX) + t.equal(writes[1].length, 1) + t.end() + }) +}) + +t.test('write() 5-arg form chunks length greater than INT32_MAX', function (t) { + writes.length = 0 + var buf = Buffer.alloc(8) + gfs.write(1, buf, 0, INT32_MAX + 5, function (err, written) { + t.error(err) + t.equal(written, INT32_MAX + 5) + t.equal(writes.length, 2) + t.equal(writes[0].length, INT32_MAX) + t.equal(writes[1].offset, INT32_MAX) + t.equal(writes[1].length, 5) + t.end() + }) +}) + +t.test('write() options form chunks length greater than INT32_MAX', function (t) { + writes.length = 0 + var buf = Buffer.alloc(8) + gfs.write(1, buf, { offset: 0, length: INT32_MAX + 1, position: null }, function (err, written) { + t.error(err) + t.equal(written, INT32_MAX + 1) + t.equal(writes.length, 2) + t.equal(writes[0].length, INT32_MAX) + t.equal(writes[1].length, 1) + t.end() + }) +}) + +t.test('writeSync() chunks length greater than INT32_MAX', function (t) { + writes.length = 0 + var buf = Buffer.alloc(8) + var written = gfs.writeSync(1, buf, 0, INT32_MAX + 1) + t.equal(written, INT32_MAX + 1) + t.equal(writes.length, 2) + t.equal(writes[0].length, INT32_MAX) + t.equal(writes[0].sync, true) + t.equal(writes[1].length, 1) + t.end() +}) + +t.test('string write is not treated as a large binary write', function (t) { + writes.length = 0 + gfs.write(1, 'hello', null, 'utf8', function (err) { + t.error(err) + t.equal(writes.length, 1) + t.end() + }) +}) + +t.test('writeFile does not pass length > INT32_MAX to write', function (t) { + var buf + try { + buf = Buffer.allocUnsafe(INT32_MAX + 1) + } catch (er) { + t.skip('cannot allocate 2GiB buffer: ' + er.message) + return t.end() + } + + writes.length = 0 + var file = path.join(t.testdir({}), 'large') + gfs.writeFile(file, buf, function (err) { + t.error(err) + t.ok(writes.length >= 2, 'split into multiple writes, got ' + writes.length) + for (var i = 0; i < writes.length; i++) { + t.ok( + writes[i].length <= INT32_MAX, + 'chunk ' + i + ' length ' + writes[i].length + ' <= INT32_MAX' + ) + } + var total = writes.reduce(function (sum, w) { return sum + w.length }, 0) + t.equal(total, INT32_MAX + 1) + t.end() + }) +})