pax_global_header00006660000000000000000000000064130576312170014517gustar00rootroot0000000000000052 comment=de157c0373a40fb5539640923cab9671cef08b12 fs-write-stream-atomic-1.0.10/000077500000000000000000000000001305763121700161015ustar00rootroot00000000000000fs-write-stream-atomic-1.0.10/.gitignore000066400000000000000000000000451305763121700200700ustar00rootroot00000000000000node_modules/ coverage/ .nyc_output/ fs-write-stream-atomic-1.0.10/.travis.yml000066400000000000000000000002151305763121700202100ustar00rootroot00000000000000language: node_js sudo: false before_install: - "npm -g install npm" node_js: - "0.8" - "0.10" - "0.12" - "iojs" - "4" - "5" fs-write-stream-atomic-1.0.10/LICENSE000066400000000000000000000013751305763121700171140ustar00rootroot00000000000000The ISC License Copyright (c) Isaac Z. Schlueter and Contributors Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. fs-write-stream-atomic-1.0.10/README.md000066400000000000000000000020111305763121700173520ustar00rootroot00000000000000# fs-write-stream-atomic Like `fs.createWriteStream(...)`, but atomic. Writes to a tmp file and does an atomic `fs.rename` to move it into place when it's done. First rule of debugging: **It's always a race condition.** ## USAGE ```javascript var fsWriteStreamAtomic = require('fs-write-stream-atomic') // options are optional. var write = fsWriteStreamAtomic('output.txt', options) var read = fs.createReadStream('input.txt') read.pipe(write) // When the write stream emits a 'finish' or 'close' event, // you can be sure that it is moved into place, and contains // all the bytes that were written to it, even if something else // was writing to `output.txt` at the same time. ``` ### `fsWriteStreamAtomic(filename, [options])` * `filename` {String} The file we want to write to * `options` {Object} * `chown` {Object} User and group to set ownership after write * `uid` {Number} * `gid` {Number} * `encoding` {String} default = 'utf8' * `mode` {Number} default = `0666` * `flags` {String} default = `'w'` fs-write-stream-atomic-1.0.10/index.js000066400000000000000000000120121305763121700175420ustar00rootroot00000000000000var fs = require('graceful-fs') var Writable = require('readable-stream').Writable var util = require('util') var MurmurHash3 = require('imurmurhash') var iferr = require('iferr') var crypto = require('crypto') function murmurhex () { var hash = MurmurHash3('') for (var ii = 0; ii < arguments.length; ++ii) { hash.hash('' + arguments[ii]) } return hash.result() } var invocations = 0 function getTmpname (filename) { return filename + '.' + murmurhex(__filename, process.pid, ++invocations) } var setImmediate = global.setImmediate || setTimeout module.exports = WriteStreamAtomic // Requirements: // 1. Write everything written to the stream to a temp file. // 2. If there are no errors: // a. moves the temp file into its final destination // b. emits `finish` & `closed` ONLY after the file is // fully flushed and renamed. // 3. If there's an error, removes the temp file. util.inherits(WriteStreamAtomic, Writable) function WriteStreamAtomic (path, options) { if (!(this instanceof WriteStreamAtomic)) { return new WriteStreamAtomic(path, options) } Writable.call(this, options) this.__isWin = options && options.hasOwnProperty('isWin') ? options.isWin : process.platform === 'win32' this.__atomicTarget = path this.__atomicTmp = getTmpname(path) this.__atomicChown = options && options.chown this.__atomicClosed = false this.__atomicStream = fs.WriteStream(this.__atomicTmp, options) this.__atomicStream.once('open', handleOpen(this)) this.__atomicStream.once('close', handleClose(this)) this.__atomicStream.once('error', handleError(this)) } // We have to suppress default finish emitting, because ordinarily it // would happen as soon as `end` is called on us and all of the // data has been written to our target stream. So we suppress // finish from being emitted here, and only emit it after our // target stream is closed and we've moved everything around. WriteStreamAtomic.prototype.emit = function (event) { if (event === 'finish') return this.__atomicStream.end() return Writable.prototype.emit.apply(this, arguments) } WriteStreamAtomic.prototype._write = function (buffer, encoding, cb) { var flushed = this.__atomicStream.write(buffer, encoding) if (flushed) return cb() this.__atomicStream.once('drain', cb) } function handleOpen (writeStream) { return function (fd) { writeStream.emit('open', fd) } } function handleClose (writeStream) { return function () { if (writeStream.__atomicClosed) return writeStream.__atomicClosed = true if (writeStream.__atomicChown) { var uid = writeStream.__atomicChown.uid var gid = writeStream.__atomicChown.gid return fs.chown(writeStream.__atomicTmp, uid, gid, iferr(cleanup, moveIntoPlace)) } else { moveIntoPlace() } } function moveIntoPlace () { fs.rename(writeStream.__atomicTmp, writeStream.__atomicTarget, iferr(trapWindowsEPERM, end)) } function trapWindowsEPERM (err) { if (writeStream.__isWin && err.syscall && err.syscall === 'rename' && err.code && err.code === 'EPERM' ) { checkFileHashes(err) } else { cleanup(err) } } function checkFileHashes (eperm) { var inprocess = 2 var tmpFileHash = crypto.createHash('sha512') var targetFileHash = crypto.createHash('sha512') fs.createReadStream(writeStream.__atomicTmp) .on('data', function (data, enc) { tmpFileHash.update(data, enc) }) .on('error', fileHashError) .on('end', fileHashComplete) fs.createReadStream(writeStream.__atomicTarget) .on('data', function (data, enc) { targetFileHash.update(data, enc) }) .on('error', fileHashError) .on('end', fileHashComplete) function fileHashError () { if (inprocess === 0) return inprocess = 0 cleanup(eperm) } function fileHashComplete () { if (inprocess === 0) return if (--inprocess) return if (tmpFileHash.digest('hex') === targetFileHash.digest('hex')) { return cleanup() } else { return cleanup(eperm) } } } function cleanup (err) { fs.unlink(writeStream.__atomicTmp, function () { if (err) { writeStream.emit('error', err) writeStream.emit('close') } else { end() } }) } function end () { // We have to use our parent class directly because we suppress `finish` // events fired via our own emit method. Writable.prototype.emit.call(writeStream, 'finish') // Delay the close to provide the same temporal separation a physical // file operation would have– that is, the close event is emitted only // after the async close operation completes. setImmediate(function () { writeStream.emit('close') }) } } function handleError (writeStream) { return function (er) { cleanupSync() writeStream.emit('error', er) writeStream.__atomicClosed = true writeStream.emit('close') } function cleanupSync () { try { fs.unlinkSync(writeStream.__atomicTmp) } finally { return } } } fs-write-stream-atomic-1.0.10/package.json000066400000000000000000000015021305763121700203650ustar00rootroot00000000000000{ "name": "fs-write-stream-atomic", "version": "1.0.10", "description": "Like `fs.createWriteStream(...)`, but atomic.", "main": "index.js", "directories": { "test": "test" }, "dependencies": { "graceful-fs": "^4.1.2", "iferr": "^0.1.5", "imurmurhash": "^0.1.4", "readable-stream": "1 || 2" }, "devDependencies": { "rimraf": "^2.4.4", "standard": "^5.4.1", "tap": "^2.3.1" }, "scripts": { "test": "standard && tap --coverage test/*.js" }, "repository": { "type": "git", "url": "https://github.com/npm/fs-write-stream-atomic" }, "author": "Isaac Z. Schlueter (http://blog.izs.me/)", "license": "ISC", "bugs": { "url": "https://github.com/npm/fs-write-stream-atomic/issues" }, "homepage": "https://github.com/npm/fs-write-stream-atomic" } fs-write-stream-atomic-1.0.10/test/000077500000000000000000000000001305763121700170605ustar00rootroot00000000000000fs-write-stream-atomic-1.0.10/test/basic.js000066400000000000000000000052661305763121700205100ustar00rootroot00000000000000var fs = require('graceful-fs') var test = require('tap').test var path = require('path') var writeStream = require('../index.js') var rename = fs.rename fs.rename = function (from, to, cb) { setTimeout(function () { rename(from, to, cb) }, 100) } test('basic', function (t) { // open 10 write streams to the same file. // then write to each of them, and to the target // and verify at the end that each of them does their thing var target = path.resolve(__dirname, 'test.txt') var n = 10 // We run all of our assertions twice: // once for finish, once for close // There are 6 assertions, two fixed, plus 4 lines in the file. t.plan(n * 2 * 6) var streams = [] for (var i = 0; i < n; i++) { var s = writeStream(target) s.on('finish', verifier('finish', i)) s.on('close', verifier('close', i)) streams.push(s) } function verifier (ev, num) { return function () { if (ev === 'close') { t.equal(this.__emittedFinish, true, num + '. closed only after finish') } else { this.__emittedFinish = true t.equal(ev, 'finish', num + '. finished') } // make sure that one of the atomic streams won. var res = fs.readFileSync(target, 'utf8') var lines = res.trim().split(/\n/) lines.forEach(function (line, lineno) { var first = lines[0].match(/\d+$/)[0] var cur = line.match(/\d+$/)[0] t.equal(cur, first, num + '. line ' + lineno + ' matches') }) var resExpr = /^first write \d+\nsecond write \d+\nthird write \d+\nfinal write \d+\n$/ t.similar(res, resExpr, num + '. content matches') } } // now write something to each stream. streams.forEach(function (stream, i) { stream.write('first write ' + i + '\n') }) // wait a sec for those writes to go out. setTimeout(function () { // write something else to the target. fs.writeFileSync(target, 'brutality!\n') // write some more stuff. streams.forEach(function (stream, i) { stream.write('second write ' + i + '\n') }) setTimeout(function () { // Oops! Deleted the file! fs.unlinkSync(target) // write some more stuff. streams.forEach(function (stream, i) { stream.write('third write ' + i + '\n') }) setTimeout(function () { fs.writeFileSync(target, 'brutality TWO!\n') streams.forEach(function (stream, i) { stream.end('final write ' + i + '\n') }) }, 50) }, 50) }, 50) }) test('cleanup', function (t) { fs.readdirSync(__dirname).filter(function (f) { return f.match(/^test.txt/) }).forEach(function (file) { fs.unlinkSync(path.resolve(__dirname, file)) }) t.end() }) fs-write-stream-atomic-1.0.10/test/chown.js000066400000000000000000000021021305763121700205270ustar00rootroot00000000000000'use strict' var fs = require('graceful-fs') var path = require('path') var test = require('tap').test var rimraf = require('rimraf') var writeStream = require('../index.js') var target = path.resolve(__dirname, 'test-chown') test('chown works', function (t) { t.plan(1) var stream = writeStream(target, {chown: {uid: process.getuid(), gid: process.getgid()}}) var hadError = false stream.on('error', function (er) { hadError = true console.log('#', er) }) stream.on('close', function () { t.is(hadError, false, 'no errors before close') }) stream.end() }) test('chown fails', function (t) { t.plan(1) fs.chown = function (file, uid, gid, cb) { cb(new Error('TEST BREAK')) } var stream = writeStream(target, {chown: {uid: process.getuid(), gid: process.getgid()}}) var hadError = false stream.on('error', function (er) { hadError = true console.log('#', er) }) stream.on('close', function () { t.is(hadError, true, 'error before close') }) stream.end() }) test('cleanup', function (t) { rimraf.sync(target) t.end() }) fs-write-stream-atomic-1.0.10/test/rename-eperm.js000066400000000000000000000101431305763121700217720ustar00rootroot00000000000000'use strict' var fs = require('graceful-fs') var path = require('path') var test = require('tap').test var rimraf = require('rimraf') var writeStream = require('../index.js') var target = path.resolve(__dirname, 'test-rename-eperm1') var target2 = path.resolve(__dirname, 'test-rename-eperm2') var target3 = path.resolve(__dirname, 'test-rename-eperm3') test('rename eperm none existing file', function (t) { t.plan(2) var _rename = fs.rename fs.existsSync = function (src) { return true } fs.rename = function (src, dest, cb) { // simulate a failure during rename where the file // is renamed successfully but the process encounters // an EPERM error and the target file does not exist _rename(src, dest, function (e) { var err = new Error('TEST BREAK') err.syscall = 'rename' err.code = 'EPERM' cb(err) }) } var stream = writeStream(target, { isWin: true }) var hadError = false var calledFinish = false stream.on('error', function (er) { hadError = true console.log('#', er) }) stream.on('finish', function () { calledFinish = true }) stream.on('close', function () { t.is(hadError, true, 'error was caught') t.is(calledFinish, false, 'finish was called before close') }) stream.end() }) // test existing file with diff. content test('rename eperm existing file different content', function (t) { t.plan(2) var _rename = fs.rename fs.existsSync = function (src) { return true } fs.rename = function (src, dest, cb) { // simulate a failure during rename where the file // is renamed successfully but the process encounters // an EPERM error and the target file that has another content than the // destination _rename(src, dest, function (e) { fs.writeFile(src, 'dest', function (writeErr) { if (writeErr) { return console.log('WRITEERR: ' + writeErr) } fs.writeFile(target2, 'target', function (writeErr) { if (writeErr) { return console.log('WRITEERR: ' + writeErr) } var err = new Error('TEST BREAK') err.syscall = 'rename' err.code = 'EPERM' cb(err) }) }) }) } var stream = writeStream(target2, { isWin: true }) var hadError = false var calledFinish = false stream.on('error', function (er) { hadError = true console.log('#', er) }) stream.on('finish', function () { calledFinish = true }) stream.on('close', function () { t.is(hadError, true, 'error was caught') t.is(calledFinish, false, 'finish was called before close') }) stream.end() }) // test existing file with the same content // test existing file with diff. content test('rename eperm existing file different content', function (t) { t.plan(2) var _rename = fs.rename fs.existsSync = function (src) { return true } fs.rename = function (src, dest, cb) { // simulate a failure during rename where the file // is renamed successfully but the process encounters // an EPERM error and the target file that has the same content than the // destination _rename(src, dest, function (e) { fs.writeFile(src, 'target2', function (writeErr) { if (writeErr) { return console.log('WRITEERR: ' + writeErr) } fs.writeFile(target3, 'target2', function (writeErr) { if (writeErr) { return console.log('WRITEERR: ' + writeErr) } var err = new Error('TEST BREAK') err.syscall = 'rename' err.code = 'EPERM' cb(err) }) }) }) } var stream = writeStream(target3, { isWin: true }) var hadError = false var calledFinish = false stream.on('error', function (er) { hadError = true console.log('#', er) }) stream.on('finish', function () { calledFinish = true }) stream.on('close', function () { t.is(hadError, false, 'error was caught') t.is(calledFinish, true, 'finish was called before close') }) stream.end() }) test('cleanup', function (t) { rimraf.sync(target) rimraf.sync(target2) rimraf.sync(target3) t.end() }) fs-write-stream-atomic-1.0.10/test/rename-fail.js000066400000000000000000000012371305763121700216010ustar00rootroot00000000000000'use strict' var fs = require('graceful-fs') var path = require('path') var test = require('tap').test var rimraf = require('rimraf') var writeStream = require('../index.js') var target = path.resolve(__dirname, 'test-rename') test('rename fails', function (t) { t.plan(1) fs.rename = function (src, dest, cb) { cb(new Error('TEST BREAK')) } var stream = writeStream(target) var hadError = false stream.on('error', function (er) { hadError = true console.log('#', er) }) stream.on('close', function () { t.is(hadError, true, 'error before close') }) stream.end() }) test('cleanup', function (t) { rimraf.sync(target) t.end() }) fs-write-stream-atomic-1.0.10/test/slow-close.js000066400000000000000000000025121305763121700215050ustar00rootroot00000000000000'use strict' var fs = require('graceful-fs') var path = require('path') var test = require('tap').test var rimraf = require('rimraf') var writeStream = require('../index.js') var target = path.resolve(__dirname, 'test-chown') test('slow close', function (t) { t.plan(2) // The goal here is to simulate the "file close" step happening so slowly // that the whole close/rename process could finish before the file is // actually closed (and thus buffers truely flushed to the OS). In // previous versions of this module, this would result in the module // emitting finish & close before the file was fully written and in // turn, could break other layers that tried to read the new file. var realEmit = fs.WriteStream.prototype.emit var reallyClosed = false fs.WriteStream.prototype.emit = function (event) { if (event !== 'close') return realEmit.apply(this, arguments) setTimeout(function () { reallyClosed = true realEmit.call(this, 'close') }.bind(this), 200) } var stream = writeStream(target) stream.on('finish', function () { t.is(reallyClosed, true, "didn't finish before target was closed") }) stream.on('close', function () { t.is(reallyClosed, true, "didn't close before target was closed") }) stream.end() }) test('cleanup', function (t) { rimraf.sync(target) t.end() }) fs-write-stream-atomic-1.0.10/test/toolong.js000066400000000000000000000012631305763121700211010ustar00rootroot00000000000000var path = require('path') var test = require('tap').test var writeStream = require('../index.js') function repeat (times, string) { var output = '' for (var ii = 0; ii < times; ++ii) { output += string } return output } var target = path.resolve(__dirname, repeat(1000, 'test')) test('name too long', function (t) { t.plan(2) var stream = writeStream(target) var hadError = false stream.on('error', function (er) { if (!hadError) { t.is(er.code, 'ENAMETOOLONG', target.length + ' character name results in ENAMETOOLONG') hadError = true } }) stream.on('close', function () { t.ok(hadError, 'got error before close') }) stream.end() })