pax_global_header 0000666 0000000 0000000 00000000064 12660610220 0014505 g ustar 00root root 0000000 0000000 52 comment=44bb895ede1645668c4f62a81c7af8edaf47bff9
unique-stream-2.2.1/ 0000775 0000000 0000000 00000000000 12660610220 0014306 5 ustar 00root root 0000000 0000000 unique-stream-2.2.1/.gitignore 0000664 0000000 0000000 00000000046 12660610220 0016276 0 ustar 00root root 0000000 0000000 *.swp
.DS_Store
coverage
node_modules
unique-stream-2.2.1/.travis.yml 0000664 0000000 0000000 00000000153 12660610220 0016416 0 ustar 00root root 0000000 0000000 sudo: false
language: node_js
node_js:
- iojs
- node
- '0.10'
after_script: npm run-script coveralls
unique-stream-2.2.1/CONTRIBUTING.md 0000664 0000000 0000000 00000002327 12660610220 0016543 0 ustar 00root root 0000000 0000000 # unique-stream is an OPEN Open Source Project
-----------------------------------------
## What?
Individuals making significant and valuable contributions are given commit-access to the project to contribute as they see fit. This project is more like an open wiki than a standard guarded open source project.
## Rules
There are a few basic ground-rules for contributors:
1. **No `--force` pushes** or modifying the Git history in any way.
1. **Non-master branches** ought to be used for ongoing work.
1. **External API changes and significant modifications** ought to be subject to an **internal pull-request** to solicit feedback from other contributors.
1. Internal pull-requests to solicit feedback are *encouraged* for any other non-trivial contribution but left to the discretion of the contributor.
1. Contributors should attempt to adhere to the prevailing code-style.
## Releases
Declaring formal releases remains the prerogative of the project maintainer.
## Changes to this arrangement
This is an experiment and feedback is welcome! This document may also be subject to pull-requests or changes by contributors where you believe you have something valuable to add or change.
-----------------------------------------
unique-stream-2.2.1/LICENSE 0000664 0000000 0000000 00000002033 12660610220 0015311 0 ustar 00root root 0000000 0000000 Copyright 2014 Eugene Ware
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
unique-stream-2.2.1/README.md 0000664 0000000 0000000 00000006672 12660610220 0015600 0 ustar 00root root 0000000 0000000 # unique-stream
node.js through stream that emits a unique stream of objects based on criteria
[](https://travis-ci.org/eugeneware/unique-stream)
[](https://coveralls.io/github/eugeneware/unique-stream?branch=master)
## Installation
Install via [npm](https://www.npmjs.com/):
```
$ npm install unique-stream
```
## Examples
### Dedupe a ReadStream based on JSON.stringify:
``` js
var unique = require('unique-stream')
, Stream = require('stream');
// return a stream of 3 identical objects
function makeStreamOfObjects() {
var s = new Stream;
s.readable = true;
var count = 3;
for (var i = 0; i < 3; i++) {
setImmediate(function () {
s.emit('data', { name: 'Bob', number: 123 });
--count || end();
});
}
function end() {
s.emit('end');
}
return s;
}
// Will only print out one object as the rest are dupes. (Uses JSON.stringify)
makeStreamOfObjects()
.pipe(unique())
.on('data', console.log);
```
### Dedupe a ReadStream based on an object property:
``` js
// Use name as the key field to dedupe on. Will only print one object
makeStreamOfObjects()
.pipe(unique('name'))
.on('data', console.log);
```
### Dedupe a ReadStream based on a custom function:
``` js
// Use a custom function to dedupe on. Use the 'number' field. Will only print one object.
makeStreamOfObjects()
.pipe(function (data) {
return data.number;
})
.on('data', console.log);
```
## Dedupe multiple streams
The reason I wrote this was to dedupe multiple object streams:
``` js
var aggregator = unique();
// Stream 1
makeStreamOfObjects()
.pipe(aggregator);
// Stream 2
makeStreamOfObjects()
.pipe(aggregator);
// Stream 3
makeStreamOfObjects()
.pipe(aggregator);
aggregator.on('data', console.log);
```
## Use a custom store to record keys that have been encountered
By default a set is used to store keys encountered so far, in order to check new ones for
uniqueness. You can supply your own store instead, providing it supports the add(key) and
has(key) methods. This could allow you to use a persistant store so that already encountered
objects are not re-streamed when node is reloaded.
``` js
var keyStore = {
store: {},
add: function(key) {
this.store[key] = true;
},
has: function(key) {
return this.store[key] !== undefined;
}
};
makeStreamOfObjects()
.pipe(unique('name', keyStore))
.on('data', console.log);
```
## Contributing
unique-stream is an **OPEN Open Source Project**. This means that:
> Individuals making significant and valuable contributions are given commit-access to the project to contribute as they see fit. This project is more like an open wiki than a standard guarded open source project.
See the [CONTRIBUTING.md](https://github.com/eugeneware/unique-stream/blob/master/CONTRIBUTING.md) file for more details.
### Contributors
unique-stream is only possible due to the excellent work of the following contributors:
unique-stream-2.2.1/index.js 0000664 0000000 0000000 00000001622 12660610220 0015754 0 ustar 00root root 0000000 0000000 'use strict';
var filter = require('through2-filter').obj;
var stringify = require("json-stable-stringify");
var ES6Set;
if (typeof global.Set === 'function') {
ES6Set = global.Set;
} else {
ES6Set = function() {
this.keys = [];
this.has = function(val) {
return this.keys.indexOf(val) !== -1;
},
this.add = function(val) {
this.keys.push(val);
}
}
}
function prop(propName) {
return function (data) {
return data[propName];
};
}
module.exports = unique;
function unique(propName, keyStore) {
keyStore = keyStore || new ES6Set();
var keyfn = stringify;
if (typeof propName === 'string') {
keyfn = prop(propName);
} else if (typeof propName === 'function') {
keyfn = propName;
}
return filter(function (data) {
var key = keyfn(data);
if (keyStore.has(key)) {
return false;
}
keyStore.add(key);
return true;
});
}
unique-stream-2.2.1/package.json 0000664 0000000 0000000 00000001452 12660610220 0016576 0 ustar 00root root 0000000 0000000 {
"name": "unique-stream",
"version": "2.2.1",
"description": "node.js through stream that emits a unique stream of objects based on criteria",
"repository": "eugeneware/unique-stream",
"author": "Eugene Ware ",
"license": "MIT",
"files": [
"index.js"
],
"scripts": {
"test": "mocha",
"coverage": "istanbul cover _mocha",
"coveralls": "${npm_package_scripts_coverage} && istanbul-coveralls"
},
"keywords": [
"unique",
"stream",
"unique-stream",
"streaming",
"streams"
],
"dependencies": {
"json-stable-stringify": "^1.0.0",
"through2-filter": "^2.0.0"
},
"devDependencies": {
"after": "~0.8.1",
"chai": "^3.0.0",
"istanbul": "^0.4.2",
"istanbul-coveralls": "^1.0.3",
"mocha": "^2.1.0"
}
}
unique-stream-2.2.1/test/ 0000775 0000000 0000000 00000000000 12660610220 0015265 5 ustar 00root root 0000000 0000000 unique-stream-2.2.1/test/index.js 0000664 0000000 0000000 00000006025 12660610220 0016735 0 ustar 00root root 0000000 0000000 var expect = require('chai').expect
, unique = require('..')
, Stream = require('stream')
, after = require('after')
, setImmediate = global.setImmediate || process.nextTick;
describe('unique stream', function() {
function makeStream(type) {
var s = new Stream();
s.readable = true;
var n = 10;
var next = after(n * 2, function () {
setImmediate(function () {
s.emit('end');
});
});
for (var k = 0; k < n * 2; k++) {
var i = Math.floor(k / 2);
var o;
if (k % 2 === 0) {
o = {
type: type,
name: 'name ' + i,
number: i * 10,
};
} else {
o = {
number: i * 10,
name: 'name ' + i,
type: type,
};
}
(function (o) {
setImmediate(function () {
s.emit('data', o);
next();
});
})(o);
}
return s;
}
it('should be able to uniqueify objects based on JSON data', function(done) {
var aggregator = unique();
makeStream('a')
.pipe(aggregator);
makeStream('a')
.pipe(aggregator);
var n = 0;
aggregator
.on('data', function () {
n++;
})
.on('end', function () {
expect(n).to.equal(10);
done();
});
});
it('should be able to uniqueify objects based on a property', function(done) {
var aggregator = unique('number');
makeStream('a')
.pipe(aggregator);
makeStream('b')
.pipe(aggregator);
var n = 0;
aggregator
.on('data', function () {
n++;
})
.on('end', function () {
expect(n).to.equal(10);
done();
});
});
it('should be able to uniqueify objects based on a function', function(done) {
var aggregator = unique(function (data) {
return data.name;
});
makeStream('a')
.pipe(aggregator);
makeStream('b')
.pipe(aggregator);
var n = 0;
aggregator
.on('data', function () {
n++;
})
.on('end', function () {
expect(n).to.equal(10);
done();
});
});
it('should be able to handle uniqueness when not piped', function(done) {
var stream = unique();
var count = 0;
stream.on('data', function (data) {
expect(data).to.equal('hello');
count++;
});
stream.on('end', function() {
expect(count).to.equal(1);
done();
});
stream.write('hello');
stream.write('hello');
stream.end();
});
it('can use a custom keystore', function(done) {
var keyStore = {
store: {},
add: function(key) {
this.store[key] = true;
},
has: function(key) {
return this.store[key] !== undefined;
}
};
var aggregator = unique('number', keyStore);
makeStream('a')
.pipe(aggregator);
makeStream('b')
.pipe(aggregator);
var n = 0;
aggregator
.on('data', function () {
n++;
})
.on('end', function () {
expect(n).to.equal(10);
done();
});
})
});