pax_global_header00006660000000000000000000000064134060704360014515gustar00rootroot0000000000000052 comment=8c20fce4c5cd58a13334a30b09a027195a93a6a9 unique-stream-2.3.1/000077500000000000000000000000001340607043600143175ustar00rootroot00000000000000unique-stream-2.3.1/.gitignore000066400000000000000000000000461340607043600163070ustar00rootroot00000000000000*.swp .DS_Store coverage node_modules unique-stream-2.3.1/.travis.yml000066400000000000000000000000721340607043600164270ustar00rootroot00000000000000sudo: false language: node_js node_js: - node - lts/* unique-stream-2.3.1/CONTRIBUTING.md000066400000000000000000000023271340607043600165540ustar00rootroot00000000000000# unique-stream is an OPEN Open Source Project ----------------------------------------- ## What? Individuals making significant and valuable contributions are given commit-access to the project to contribute as they see fit. This project is more like an open wiki than a standard guarded open source project. ## Rules There are a few basic ground-rules for contributors: 1. **No `--force` pushes** or modifying the Git history in any way. 1. **Non-master branches** ought to be used for ongoing work. 1. **External API changes and significant modifications** ought to be subject to an **internal pull-request** to solicit feedback from other contributors. 1. Internal pull-requests to solicit feedback are *encouraged* for any other non-trivial contribution but left to the discretion of the contributor. 1. Contributors should attempt to adhere to the prevailing code-style. ## Releases Declaring formal releases remains the prerogative of the project maintainer. ## Changes to this arrangement This is an experiment and feedback is welcome! This document may also be subject to pull-requests or changes by contributors where you believe you have something valuable to add or change. ----------------------------------------- unique-stream-2.3.1/LICENSE000066400000000000000000000020331340607043600153220ustar00rootroot00000000000000Copyright 2014 Eugene Ware Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. unique-stream-2.3.1/README.md000066400000000000000000000070611340607043600156020ustar00rootroot00000000000000# unique-stream node.js through stream that emits a unique stream of objects based on criteria [![Build Status](https://travis-ci.org/eugeneware/unique-stream.svg?branch=master)](https://travis-ci.org/eugeneware/unique-stream) [![Coverage Status](https://coveralls.io/repos/eugeneware/unique-stream/badge.svg?branch=master&service=github)](https://coveralls.io/github/eugeneware/unique-stream?branch=master) ## Installation Install via [npm](https://www.npmjs.com/): ``` $ npm install unique-stream ``` ## Examples ### Dedupe a ReadStream based on JSON.stringify: ``` js var unique = require('unique-stream') , Stream = require('stream'); // return a stream of 3 identical objects function makeStreamOfObjects() { var s = new Stream; s.readable = true; var count = 3; for (var i = 0; i < 3; i++) { setImmediate(function () { s.emit('data', { name: 'Bob', number: 123 }); --count || end(); }); } function end() { s.emit('end'); } return s; } // Will only print out one object as the rest are dupes. (Uses JSON.stringify) makeStreamOfObjects() .pipe(unique()) .on('data', console.log); ``` ### Dedupe a ReadStream based on an object property: ``` js // Use name as the key field to dedupe on. Will only print one object makeStreamOfObjects() .pipe(unique('name')) .on('data', console.log); ``` ### Dedupe a ReadStream based on a custom function: ``` js // Use a custom function to dedupe on. Use the 'number' field. Will only print one object. makeStreamOfObjects() .pipe(function (data) { return data.number; }) .on('data', console.log); ``` ## Dedupe multiple streams The reason I wrote this was to dedupe multiple object streams: ``` js var aggregator = unique(); // Stream 1 makeStreamOfObjects() .pipe(aggregator); // Stream 2 makeStreamOfObjects() .pipe(aggregator); // Stream 3 makeStreamOfObjects() .pipe(aggregator); aggregator.on('data', console.log); ``` ## Use a custom store to record keys that have been encountered By default a set is used to store keys encountered so far, in order to check new ones for uniqueness. You can supply your own store instead, providing it supports the add(key) and has(key) methods. This could allow you to use a persistent store so that already encountered objects are not re-streamed when node is reloaded. ``` js var keyStore = { store: {}, add: function(key) { this.store[key] = true; }, has: function(key) { return this.store[key] !== undefined; } }; makeStreamOfObjects() .pipe(unique('name', keyStore)) .on('data', console.log); ``` ## Contributing unique-stream is an **OPEN Open Source Project**. This means that: > Individuals making significant and valuable contributions are given commit-access to the project to contribute as they see fit. This project is more like an open wiki than a standard guarded open source project. See the [CONTRIBUTING.md](https://github.com/eugeneware/unique-stream/blob/master/CONTRIBUTING.md) file for more details. ### Contributors unique-stream is only possible due to the excellent work of the following contributors:
Eugene WareGitHub/eugeneware
Craig AmbroseGitHub/craigambrose
Shinnosuke WatanabeGitHub/shinnn
Rouven WeßlingGitHub/realityking
unique-stream-2.3.1/index.js000066400000000000000000000016421340607043600157670ustar00rootroot00000000000000'use strict'; var filter = require('through2-filter').obj; var stringify = require("json-stable-stringify-without-jsonify"); var ES6Set; if (typeof global.Set === 'function') { ES6Set = global.Set; } else { ES6Set = function() { this.keys = []; this.has = function(val) { return this.keys.indexOf(val) !== -1; }, this.add = function(val) { this.keys.push(val); } } } function prop(propName) { return function (data) { return data[propName]; }; } module.exports = unique; function unique(propName, keyStore) { keyStore = keyStore || new ES6Set(); var keyfn = stringify; if (typeof propName === 'string') { keyfn = prop(propName); } else if (typeof propName === 'function') { keyfn = propName; } return filter(function (data) { var key = keyfn(data); if (keyStore.has(key)) { return false; } keyStore.add(key); return true; }); } unique-stream-2.3.1/package-lock.json000066400000000000000000000542221340607043600175400ustar00rootroot00000000000000{ "name": "unique-stream", "version": "2.3.1", "lockfileVersion": 1, "requires": true, "dependencies": { "abbrev": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", "integrity": "sha1-kbR5JYinc4wl813W9jdSovh3YTU=", "dev": true }, "after": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz", "integrity": "sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8=", "dev": true }, "amdefine": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", "dev": true, "optional": true }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, "requires": { "sprintf-js": "~1.0.2" } }, "assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", "dev": true }, "async": { "version": "1.5.2", "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, "chai": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", "integrity": "sha512-XQU3bhBukrOsQCuwZndwGcCVQHyZi53fQ6Ys1Fym7E4olpIqqZZhhoFJoaKVvV17lWQoXYwgWN2nF5crA8J2jw==", "dev": true, "requires": { "assertion-error": "^1.1.0", "check-error": "^1.0.2", "deep-eql": "^3.0.1", "get-func-name": "^2.0.0", "pathval": "^1.1.0", "type-detect": "^4.0.5" } }, "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, "commander": { "version": "2.17.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==", "dev": true, "optional": true }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "dev": true, "requires": { "ms": "2.0.0" } }, "deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", "dev": true, "requires": { "type-detect": "^4.0.0" } }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, "diff": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, "escodegen": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.8.1.tgz", "integrity": "sha1-WltTr0aTEQvrsIZ6o0MN07cKEBg=", "dev": true, "requires": { "esprima": "^2.7.1", "estraverse": "^1.9.1", "esutils": "^2.0.2", "optionator": "^0.8.1", "source-map": "~0.2.0" } }, "esprima": { "version": "2.7.3", "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", "dev": true }, "estraverse": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz", "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=", "dev": true }, "esutils": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", "dev": true }, "fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", "dev": true }, "glob": { "version": "5.0.15", "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=", "dev": true, "requires": { "inflight": "^1.0.4", "inherits": "2", "minimatch": "2 || 3", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "growl": { "version": "1.10.5", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, "handlebars": { "version": "4.0.12", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.12.tgz", "integrity": "sha512-RhmTekP+FZL+XNhwS1Wf+bTTZpdLougwt5pcgA1tuz6Jcx0fpH/7z0qd71RKnZHBCxIRBHfBOnio4gViPemNzA==", "dev": true, "requires": { "async": "^2.5.0", "optimist": "^0.6.1", "source-map": "^0.6.1", "uglify-js": "^3.1.4" }, "dependencies": { "async": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", "dev": true, "requires": { "lodash": "^4.17.10" } }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true } } }, "has-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=", "dev": true }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", "dev": true }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" } }, "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, "istanbul": { "version": "0.4.5", "resolved": "https://registry.npmjs.org/istanbul/-/istanbul-0.4.5.tgz", "integrity": "sha1-ZcfXPUxNqE1POsMQuRj7C4Azczs=", "dev": true, "requires": { "abbrev": "1.0.x", "async": "1.x", "escodegen": "1.8.x", "esprima": "2.7.x", "glob": "^5.0.15", "handlebars": "^4.0.1", "js-yaml": "3.x", "mkdirp": "0.5.x", "nopt": "3.x", "once": "1.x", "resolve": "1.1.x", "supports-color": "^3.1.0", "which": "^1.1.1", "wordwrap": "^1.0.0" } }, "js-yaml": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz", "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==", "dev": true, "requires": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "dependencies": { "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dev": true } } }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=" }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", "dev": true, "requires": { "prelude-ls": "~1.1.2", "type-check": "~0.3.2" } }, "lodash": { "version": "4.17.11", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", "dev": true }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { "version": "0.0.10", "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", "dev": true }, "mkdirp": { "version": "0.5.1", "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "requires": { "minimist": "0.0.8" }, "dependencies": { "minimist": { "version": "0.0.8", "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", "dev": true } } }, "mocha": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", "dev": true, "requires": { "browser-stdout": "1.3.1", "commander": "2.15.1", "debug": "3.1.0", "diff": "3.5.0", "escape-string-regexp": "1.0.5", "glob": "7.1.2", "growl": "1.10.5", "he": "1.1.1", "minimatch": "3.0.4", "mkdirp": "0.5.1", "supports-color": "5.4.0" }, "dependencies": { "commander": { "version": "2.15.1", "resolved": "http://registry.npmjs.org/commander/-/commander-2.15.1.tgz", "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", "dev": true }, "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.0.4", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", "dev": true }, "supports-color": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", "dev": true, "requires": { "has-flag": "^3.0.0" } } } }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, "nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", "dev": true, "requires": { "abbrev": "1" } }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, "requires": { "wrappy": "1" } }, "optimist": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", "dev": true, "requires": { "minimist": "~0.0.1", "wordwrap": "~0.0.2" }, "dependencies": { "wordwrap": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", "dev": true } } }, "optionator": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", "dev": true, "requires": { "deep-is": "~0.1.3", "fast-levenshtein": "~2.0.4", "levn": "~0.3.0", "prelude-ls": "~1.1.2", "type-check": "~0.3.2", "wordwrap": "~1.0.0" } }, "path-is-absolute": { "version": "1.0.1", "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, "pathval": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", "dev": true }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, "process-nextick-args": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, "readable-stream": { "version": "2.3.6", "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "resolve": { "version": "1.1.7", "resolved": "http://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=", "dev": true }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "source-map": { "version": "0.2.0", "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.2.0.tgz", "integrity": "sha1-2rc/vPwrqBm03gO9b26qSBZLP50=", "dev": true, "optional": true, "requires": { "amdefine": ">=0.0.4" } }, "sprintf-js": { "version": "1.0.3", "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, "string_decoder": { "version": "1.1.1", "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" } }, "supports-color": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz", "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=", "dev": true, "requires": { "has-flag": "^1.0.0" } }, "through2": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", "requires": { "readable-stream": "~2.3.6", "xtend": "~4.0.1" } }, "through2-filter": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/through2-filter/-/through2-filter-3.0.0.tgz", "integrity": "sha512-jaRjI2WxN3W1V8/FMZ9HKIBXixtiqs3SQSX4/YGIiP3gL6djW48VoZq9tDqeCWs3MT8YY5wb/zli8VW8snY1CA==", "requires": { "through2": "~2.0.0", "xtend": "~4.0.0" } }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", "dev": true, "requires": { "prelude-ls": "~1.1.2" } }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", "dev": true }, "uglify-js": { "version": "3.4.9", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.9.tgz", "integrity": "sha512-8CJsbKOtEbnJsTyv6LE6m6ZKniqMiFWmm9sRbopbkGs3gMPPfd3Fh8iIA4Ykv5MgaTbqHr4BaoGLJLZNhsrW1Q==", "dev": true, "optional": true, "requires": { "commander": "~2.17.1", "source-map": "~0.6.1" }, "dependencies": { "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true, "optional": true } } }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "requires": { "isexe": "^2.0.0" } }, "wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", "dev": true }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true }, "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" } } } unique-stream-2.3.1/package.json000066400000000000000000000013141340607043600166040ustar00rootroot00000000000000{ "name": "unique-stream", "version": "2.3.1", "description": "node.js through stream that emits a unique stream of objects based on criteria", "repository": "eugeneware/unique-stream", "author": "Eugene Ware ", "license": "MIT", "files": [ "index.js" ], "scripts": { "test": "mocha", "coverage": "istanbul cover _mocha" }, "keywords": [ "unique", "stream", "unique-stream", "streaming", "streams" ], "dependencies": { "json-stable-stringify-without-jsonify": "^1.0.1", "through2-filter": "^3.0.0" }, "devDependencies": { "after": "~0.8.1", "chai": "^4.2.0", "istanbul": "^0.4.5", "mocha": "^5.2.0" } } unique-stream-2.3.1/test/000077500000000000000000000000001340607043600152765ustar00rootroot00000000000000unique-stream-2.3.1/test/index.js000066400000000000000000000060251340607043600167460ustar00rootroot00000000000000var expect = require('chai').expect , unique = require('..') , Stream = require('stream') , after = require('after') , setImmediate = global.setImmediate || process.nextTick; describe('unique stream', function() { function makeStream(type) { var s = new Stream(); s.readable = true; var n = 10; var next = after(n * 2, function () { setImmediate(function () { s.emit('end'); }); }); for (var k = 0; k < n * 2; k++) { var i = Math.floor(k / 2); var o; if (k % 2 === 0) { o = { type: type, name: 'name ' + i, number: i * 10, }; } else { o = { number: i * 10, name: 'name ' + i, type: type, }; } (function (o) { setImmediate(function () { s.emit('data', o); next(); }); })(o); } return s; } it('should be able to uniqueify objects based on JSON data', function(done) { var aggregator = unique(); makeStream('a') .pipe(aggregator); makeStream('a') .pipe(aggregator); var n = 0; aggregator .on('data', function () { n++; }) .on('end', function () { expect(n).to.equal(10); done(); }); }); it('should be able to uniqueify objects based on a property', function(done) { var aggregator = unique('number'); makeStream('a') .pipe(aggregator); makeStream('b') .pipe(aggregator); var n = 0; aggregator .on('data', function () { n++; }) .on('end', function () { expect(n).to.equal(10); done(); }); }); it('should be able to uniqueify objects based on a function', function(done) { var aggregator = unique(function (data) { return data.name; }); makeStream('a') .pipe(aggregator); makeStream('b') .pipe(aggregator); var n = 0; aggregator .on('data', function () { n++; }) .on('end', function () { expect(n).to.equal(10); done(); }); }); it('should be able to handle uniqueness when not piped', function(done) { var stream = unique(); var count = 0; stream.on('data', function (data) { expect(data).to.equal('hello'); count++; }); stream.on('end', function() { expect(count).to.equal(1); done(); }); stream.write('hello'); stream.write('hello'); stream.end(); }); it('can use a custom keystore', function(done) { var keyStore = { store: {}, add: function(key) { this.store[key] = true; }, has: function(key) { return this.store[key] !== undefined; } }; var aggregator = unique('number', keyStore); makeStream('a') .pipe(aggregator); makeStream('b') .pipe(aggregator); var n = 0; aggregator .on('data', function () { n++; }) .on('end', function () { expect(n).to.equal(10); done(); }); }) });