pax_global_header00006660000000000000000000000064130103127120014500gustar00rootroot0000000000000052 comment=1b4919bc318eb0cbd6e8ee08c4d56f405d74e643 parallel-transform-1.1.0/000077500000000000000000000000001301031271200153045ustar00rootroot00000000000000parallel-transform-1.1.0/.gitignore000066400000000000000000000000151301031271200172700ustar00rootroot00000000000000node_modules parallel-transform-1.1.0/LICENSE000066400000000000000000000020341301031271200163100ustar00rootroot00000000000000Copyright 2013 Mathias Buus Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. parallel-transform-1.1.0/README.md000066400000000000000000000026241301031271200165670ustar00rootroot00000000000000# parallel-transform [Transform stream](http://nodejs.org/api/stream.html#stream_class_stream_transform_1) for Node.js that allows you to run your transforms in parallel without changing the order of the output. npm install parallel-transform It is easy to use ``` js var transform = require('parallel-transform'); var stream = transform(10, function(data, callback) { // 10 is the parallism level setTimeout(function() { callback(null, data); }, Math.random() * 1000); }); for (var i = 0; i < 10; i++) { stream.write(''+i); } stream.end(); stream.on('data', function(data) { console.log(data); // prints 0,1,2,... }); stream.on('end', function() { console.log('stream has ended'); }); ``` If you run the above example you'll notice that it runs in parallel (does not take ~1 second between each print) and that the order is preserved ## Stream options All transforms are Node 0.10 streams. Per default they are created with the options `{objectMode:true}`. If you want to use your own stream options pass them as the second parameter ``` js var stream = transform(10, {objectMode:false}, function(data, callback) { // data is now a buffer callback(null, data); }); fs.createReadStream('filename').pipe(stream).pipe(process.stdout); ``` ### Unordered Passing the option `{ordered:false}` will output the data as soon as it's processed by a transform, without waiting to respect the order. ## License MITparallel-transform-1.1.0/index.js000066400000000000000000000047521301031271200167610ustar00rootroot00000000000000var Transform = require('readable-stream').Transform; var inherits = require('inherits'); var cyclist = require('cyclist'); var util = require('util'); var ParallelTransform = function(maxParallel, opts, ontransform) { if (!(this instanceof ParallelTransform)) return new ParallelTransform(maxParallel, opts, ontransform); if (typeof maxParallel === 'function') { ontransform = maxParallel; opts = null; maxParallel = 1; } if (typeof opts === 'function') { ontransform = opts; opts = null; } if (!opts) opts = {}; if (!opts.highWaterMark) opts.highWaterMark = Math.max(maxParallel, 16); if (opts.objectMode !== false) opts.objectMode = true; Transform.call(this, opts); this._maxParallel = maxParallel; this._ontransform = ontransform; this._destroyed = false; this._flushed = false; this._ordered = opts.ordered !== false; this._buffer = this._ordered ? cyclist(maxParallel) : []; this._top = 0; this._bottom = 0; this._ondrain = null; }; inherits(ParallelTransform, Transform); ParallelTransform.prototype.destroy = function() { if (this._destroyed) return; this._destroyed = true; this.emit('close'); }; ParallelTransform.prototype._transform = function(chunk, enc, callback) { var self = this; var pos = this._top++; this._ontransform(chunk, function(err, data) { if (self._destroyed) return; if (err) { self.emit('error', err); self.push(null); self.destroy(); return; } if (self._ordered) { self._buffer.put(pos, (data === undefined || data === null) ? null : data); } else { self._buffer.push(data); } self._drain(); }); if (this._top - this._bottom < this._maxParallel) return callback(); this._ondrain = callback; }; ParallelTransform.prototype._flush = function(callback) { this._flushed = true; this._ondrain = callback; this._drain(); }; ParallelTransform.prototype._drain = function() { if (this._ordered) { while (this._buffer.get(this._bottom) !== undefined) { var data = this._buffer.del(this._bottom++); if (data === null) continue; this.push(data); } } else { while (this._buffer.length > 0) { var data = this._buffer.pop(); this._bottom++; if (data === null) continue; this.push(data); } } if (!this._drained() || !this._ondrain) return; var ondrain = this._ondrain; this._ondrain = null; ondrain(); }; ParallelTransform.prototype._drained = function() { var diff = this._top - this._bottom; return this._flushed ? !diff : diff < this._maxParallel; }; module.exports = ParallelTransform; parallel-transform-1.1.0/package.json000066400000000000000000000010101301031271200175620ustar00rootroot00000000000000{ "name": "parallel-transform", "version": "1.1.0", "repository": "git://github.com/mafintosh/parallel-transform", "license": "MIT", "description": "Transform stream that allows you to run your transforms in parallel without changing the order", "keywords": [ "transform", "stream", "parallel", "preserve", "order" ], "author": "Mathias Buus Madsen ", "dependencies": { "cyclist": "~0.2.2", "inherits": "^2.0.3", "readable-stream": "^2.1.5" } }