pax_global_header00006660000000000000000000000064117136014110014505gustar00rootroot0000000000000052 comment=040d94e07461aea6567ea581e8a65fa44bfc3cc4 felixge-node-dirty-376f4b7/000077500000000000000000000000001171360141100155505ustar00rootroot00000000000000felixge-node-dirty-376f4b7/.gitignore000066400000000000000000000000241171360141100175340ustar00rootroot00000000000000*.dirty node_modulesfelixge-node-dirty-376f4b7/LICENSE.txt000066400000000000000000000020741171360141100173760ustar00rootroot00000000000000Copyright (c) 2010 Debuggable Limited Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.felixge-node-dirty-376f4b7/Makefile000066400000000000000000000006301171360141100172070ustar00rootroot00000000000000test: @find test/{simple,system}/test-*.js | xargs -n 1 -t node benchmark-v8: @find benchmark/v8/*.js | xargs -n 1 -t node benchmark-php: @find benchmark/php/*.php | xargs -n 1 -t php benchmark-dirty: @find benchmark/dirty/*.js | xargs -n 1 -t node benchmark-all: benchmark-v8 benchmark-php benchmark-dirty benchmark: benchmark-dirty clean: @find . -name *.dirty | xargs rm .PHONY: test benchmark felixge-node-dirty-376f4b7/README.md000066400000000000000000000056321171360141100170350ustar00rootroot00000000000000# node-dirty ## Purpose A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records. ## Installation npm install dirty ## Why dirty? This module is called dirty because: * The file format is newline separated JSON * Your database lives in the same process as your application, they share memory * There is no query language, you just `forEach` through all records So dirty means that you will hit a very hard wall with this database after ~1 million records, but it is a wonderful solution for anything smaller than that. ## Tutorial require('../test/common'); var db = require('dirty')('user.db'); db.on('load', function() { db.set('john', {eyes: 'blue'}); console.log('Added john, he has %s eyes.', db.get('john').eyes); db.set('bob', {eyes: 'brown'}, function() { console.log('User bob is now saved on disk.') }); db.forEach(function(key, val) { console.log('Found key: %s, val: %j', key, val); }); }); db.on('drain', function() { console.log('All records are saved on disk now.'); }); Output: Added john, he has blue eyes. Found key: john, val: {"eyes":"blue"} Found key: bob, val: {"eyes":"brown"} User bob is now saved on disk. All records are saved on disk now. ## API ### new Dirty([path]) Creates a new dirty database. If `path` does not exist yet, it is created. You can also omit the `path` if you don't want disk persistence (useful for testing). The constructor can be invoked in multiple ways: require('dirty')('my.db'); require('dirty').Dirty('my.db'); new (require('dirty'))('my.db'); new (require('dirty').Dirty)('my.db'); ### dirty.path The path of the dirty database. ### dirty.set(key, value, [cb]) Set's the given `key` / `val` pair. The state of the database is affected instantly, the optional `cb` callback is fired when the record was written to disk. `val` can be any JSON-serializable type, it does not have to be an object. ### dirty.get(key) Retrieves the value for the given `key`. ### dirty.rm(key, cb) Removes the record with the given `key`. This is identical to setting the `key`'s value to `undefined`. ### dirty.forEach(fn) Calls the given `fn` function for every document in the database. The passed arguments are `key` and `val`. You can return `false` to abort a query (useful if you are only interested in a limited number of records). This function is blocking and runs at ~4 Mhz. ### dirty event: 'load' (length) Emitted once the database file has finished loading. It is not safe to access records before this event fires. Writing records however should be fine. `length` is the amount of records the database is holding. This only counts each key once, even if it had been overwritten. ### dirty event: 'drain' () Emitted whenever all records have been written to disk. ## License node-dirty is licensed under the MIT license. felixge-node-dirty-376f4b7/benchmark/000077500000000000000000000000001171360141100175025ustar00rootroot00000000000000felixge-node-dirty-376f4b7/benchmark/dirty/000077500000000000000000000000001171360141100206355ustar00rootroot00000000000000felixge-node-dirty-376f4b7/benchmark/dirty/for-each.js000066400000000000000000000010541171360141100226570ustar00rootroot00000000000000require('../../test/common'); var COUNT = 1e6, dirty = require('dirty')(), util = require('util'); for (var i = 0; i < COUNT; i++) { dirty.set(i, i); } var start = +new Date, i = 0; dirty.forEach(function(key, doc) { if (!key && key !== 0) { throw new Error('implementation fail'); } }); var ms = +new Date - start, mhz = ((COUNT / (ms / 1000)) / 1e6).toFixed(2), million = COUNT / 1e6; // Can't use console.log() since since I also test this in ancient node versions util.log(mhz+' Mhz ('+million+' million in '+ms+' ms)'); felixge-node-dirty-376f4b7/benchmark/dirty/get.js000066400000000000000000000010431171360141100217500ustar00rootroot00000000000000require('../../test/common'); var COUNT = 1e6, dirty = require('dirty')(), util = require('util'); for (var i = 0; i < COUNT; i++) { dirty.set(i, i); } var start = +new Date; for (var i = 0; i < COUNT; i++) { if (dirty.get(i) !== i) { throw new Error('implementation fail'); } } var ms = +new Date - start, mhz = ((COUNT / (ms / 1000)) / 1e6).toFixed(2), million = COUNT / 1e6; // Can't use console.log() since since I also test this in ancient node versions util.log(mhz+' Mhz ('+million+' million in '+ms+' ms)'); felixge-node-dirty-376f4b7/benchmark/dirty/load.js000066400000000000000000000013711171360141100221140ustar00rootroot00000000000000require('../../test/common'); var COUNT = 1e4, DB_FILE = __dirname+'/../../test/tmp/benchmark-set-drain.dirty', dirty = require('dirty')(DB_FILE), util = require('util'), loaded = false; for (var i = 0; i < COUNT; i++) { dirty.set(i, i); } dirty.on('drain', function() { var start = +new Date; require('dirty')(DB_FILE).on('load', function(length) { var ms = +new Date - start, mhz = ((COUNT / (ms / 1000)) / 1e3).toFixed(2), million = COUNT / 1e6; // Can't use console.log() since since I also test this in ancient node versions util.log(mhz+' Hz ('+million+' million in '+ms+' ms)'); loaded = true; assert.equal(length, COUNT); }); }); process.on('exit', function() { assert.ok(loaded); }); felixge-node-dirty-376f4b7/benchmark/dirty/set-drain-256-bytes-per-doc.js000066400000000000000000000015661171360141100260560ustar00rootroot00000000000000require('../../test/common'); var COUNT = 1e5, dirty = require('dirty')(__dirname+'/../../test/tmp/benchmark-set-drain.dirty'), util = require('util'), drained = false; var start = +new Date; for (var i = 0; i < COUNT; i++) { dirty.set(i, 'This string has 256 bytes. This string has 256 bytes. This string has 256 bytes. This string has 256 bytes. This string has 256 bytes. This string has 256 bytes. This string has 256 bytes. This string has 256 bytes. This string has 256 bytes. This string'); } dirty.on('drain', function() { var ms = +new Date - start, mhz = ((COUNT / (ms / 1000)) / 1e3).toFixed(2), million = COUNT / 1e6; // Can't use console.log() since since I also test this in ancient node versions util.log(mhz+' Hz ('+million+' million in '+ms+' ms)'); drained = true; }); process.on('exit', function() { assert.ok(drained); }); felixge-node-dirty-376f4b7/benchmark/dirty/set-drain.js000066400000000000000000000011651171360141100230640ustar00rootroot00000000000000require('../../test/common'); var COUNT = 1e4, dirty = require('dirty')(__dirname+'/../../test/tmp/benchmark-set-drain.dirty'), util = require('util'), drained = false; var start = +new Date; for (var i = 0; i < COUNT; i++) { dirty.set(i, i); } dirty.on('drain', function() { var ms = +new Date - start, mhz = ((COUNT / (ms / 1000)) / 1e3).toFixed(2), million = COUNT / 1e6; // Can't use console.log() since since I also test this in ancient node versions util.log(mhz+' Hz ('+million+' million in '+ms+' ms)'); drained = true; }); process.on('exit', function() { assert.ok(drained); }); felixge-node-dirty-376f4b7/benchmark/dirty/set.js000066400000000000000000000007621171360141100217730ustar00rootroot00000000000000require('../../test/common'); var COUNT = 1e6, dirty = require('dirty')(__dirname+'/../../test/tmp/benchmark-set.dirty'), util = require('util'); var start = +new Date; for (var i = 0; i < COUNT; i++) { dirty.set(i, i); } var ms = +new Date - start, mhz = ((COUNT / (ms / 1000)) / 1e6).toFixed(2), million = COUNT / 1e6; // Can't use console.log() since since I also test this in ancient node versions util.log(mhz+' Mhz ('+million+' million in '+ms+' ms)'); process.exit(0); felixge-node-dirty-376f4b7/benchmark/php/000077500000000000000000000000001171360141100202715ustar00rootroot00000000000000felixge-node-dirty-376f4b7/benchmark/php/array-get.php000066400000000000000000000006421171360141100226770ustar00rootroot00000000000000=0.8.0" }, "engines": { "node": "*" } }felixge-node-dirty-376f4b7/test/000077500000000000000000000000001171360141100165275ustar00rootroot00000000000000felixge-node-dirty-376f4b7/test/common.js000066400000000000000000000007361171360141100203630ustar00rootroot00000000000000var path = require('path'), fs = require('fs'); global.ROOT_DIR = path.dirname(__dirname); global.TEST_TMP = path.join(__dirname, 'tmp'); global.ROOT_LIB = path.join(global.ROOT_DIR, 'lib', 'dirty'); global.assert = require('assert'); global.Gently = require('gently'); global.GENTLY = new Gently(); global.HIJACKED = GENTLY.hijacked; fs.readdirSync(TEST_TMP).forEach(function(file) { if (!file.match(/\.dirty$/)) { return; } fs.unlinkSync(TEST_TMP+'/'+file); }); felixge-node-dirty-376f4b7/test/nostore/000077500000000000000000000000001171360141100202205ustar00rootroot00000000000000felixge-node-dirty-376f4b7/test/nostore/test-load-event.js000066400000000000000000000003461171360141100235740ustar00rootroot00000000000000require('../common'); var assert = require('assert'); var dirty = require('dirty')(''); var isLoaded = false; dirty.on('load', function() { isLoaded = true; }); setTimeout(function() { assert.equal(isLoaded, true); }, 500);felixge-node-dirty-376f4b7/test/nostore/test-set-callback.js000066400000000000000000000003471171360141100240640ustar00rootroot00000000000000require('../common'); var assert = require('assert'); var dirty = require('dirty')(''); var foo = ''; dirty.set('foo', 'bar', function() { foo = dirty.get('foo'); }); setTimeout(function() { assert.equal(foo, 'bar'); }, 500);felixge-node-dirty-376f4b7/test/simple/000077500000000000000000000000001171360141100200205ustar00rootroot00000000000000felixge-node-dirty-376f4b7/test/simple/test-dirty.js000066400000000000000000000200531171360141100224660ustar00rootroot00000000000000require('../common'); var Dirty = require('dirty'), EventEmitter = require('events').EventEmitter, dirtyLoad = Dirty.prototype._load, gently, dirty; (function testConstructor() { var gently = new Gently(); (function testBasic() { var PATH = '/foo/bar'; Dirty.prototype._load = gently.expect(function() { assert.equal(this.path, PATH); }); var dirty = new Dirty(PATH); assert.ok(dirty instanceof EventEmitter); assert.deepEqual(dirty._docs, {}); assert.deepEqual(dirty._queue, []); assert.strictEqual(dirty.writeBundle, 1000); assert.strictEqual(dirty._writeStream, null); assert.strictEqual(dirty._readStream, null); })(); (function testWithoutNew() { Dirty.prototype._load = gently.expect(function() {}); var dirty = Dirty(); })(); (function testOldSchoolClassName() { assert.strictEqual(Dirty, Dirty.Dirty); })(); Dirty.prototype._load = function(){}; gently.verify(); })(); function test(fn) { gently = new Gently(); dirty = Dirty(); fn(); gently.verify(); } test(function _load() { (function testNoPath() { gently.expect(HIJACKED.fs, 'createWriteStream', 0); dirtyLoad.call(dirty); })(); (function testWithPath() { var PATH = dirty.path = '/dirty.db', READ_STREAM = {}, WRITE_STREAM = {}, readStreamEmit = {}; gently.expect(HIJACKED.fs, 'createReadStream', function (path, options) { assert.equal(path, PATH); assert.equal(options.flags, 'r'); assert.equal(options.encoding, 'utf-8'); return READ_STREAM; }); var EVENTS = ['error', 'data', 'end']; gently.expect(READ_STREAM, 'on', EVENTS.length, function (event, cb) { assert.strictEqual(event, EVENTS.shift()); readStreamEmit[event] = cb; return this; }); gently.expect(HIJACKED.fs, 'createWriteStream', function (path, options) { assert.equal(path, PATH); assert.equal(options.flags, 'a'); assert.equal(options.encoding, 'utf-8'); return WRITE_STREAM; }); gently.expect(WRITE_STREAM, 'on', function (event, cb) { assert.strictEqual(event, 'drain'); (function testQueueEmpty() { dirty._queue = []; dirty.flushing = true; gently.expect(dirty, 'emit', function (event) { assert.strictEqual(event, 'drain'); }); cb(); assert.strictEqual(dirty.flushing, false); })(); (function testQueueNotEmpty() { dirty._queue = [1]; dirty.flushing = true; gently.expect(dirty, '_maybeFlush'); cb(); assert.strictEqual(dirty.flushing, false); })(); }); dirtyLoad.call(dirty); assert.strictEqual(dirty._writeStream, WRITE_STREAM); assert.strictEqual(dirty._readStream, READ_STREAM); (function testReading() { readStreamEmit.data( JSON.stringify({key: 1, val: 'A'})+'\n'+ JSON.stringify({key: 2, val: 'B'})+'\n' ); assert.equal(dirty.get(1), 'A'); assert.equal(dirty.get(2), 'B'); readStreamEmit.data('{"key": 3'); readStreamEmit.data(', "val": "C"}\n'); assert.equal(dirty.get(3), 'C'); readStreamEmit.data( JSON.stringify({key: 3, val: 'C2'})+'\n'+ JSON.stringify({key: 4, val: undefined})+'\n' ); gently.expect(dirty, 'emit', function (event, err) { assert.equal(event, 'error'); assert.equal(err.message, 'Could not load corrupted row: {broken'); }); readStreamEmit.data('{broken\n'); gently.expect(dirty, 'emit', function (event, err) { assert.equal(event, 'error'); assert.equal(err.message, 'Could not load corrupted row: {}'); }); readStreamEmit.data('{}\n'); readStreamEmit.data( JSON.stringify({key: 1, val: undefined})+'\n' ); assert.ok(!('1' in dirty._docs)); })(); (function testReadEnd() { gently.expect(dirty, 'emit', function (event, length) { assert.equal(event, 'load'); assert.equal(length, 2); }); readStreamEmit.end(); })(); (function testReadEndWithStuffLeftInBuffer() { readStreamEmit.data('foo'); gently.expect(dirty, 'emit', function (event, err) { assert.equal(event, 'error'); assert.equal(err.message, 'Corrupted row at the end of the db: foo'); }); gently.expect(dirty, 'emit', function (event) { assert.equal(event, 'load'); }); readStreamEmit.end(); })(); (function testReadDbError() { var ERR = new Error('oh oh'); gently.expect(dirty, 'emit', function (event, err) { assert.equal(event, 'error'); assert.strictEqual(err, ERR); }); readStreamEmit.error(ERR) })(); (function testReadNonexistingDbError() { gently.expect(dirty, 'emit', function (event, length) { assert.equal(event, 'load'); assert.equal(length, 0); }); readStreamEmit.error({ code: 'ENOENT' }) })(); })(); }); test(function get() { var KEY = 'example', VAL = {}; dirty._docs[KEY] = VAL; assert.strictEqual(dirty.get(KEY), VAL); }); test(function set() { (function testNoCallback() { var KEY = 'example', VAL = {}; gently.expect(dirty, '_maybeFlush'); dirty.set(KEY, VAL); assert.strictEqual(dirty._docs[KEY], VAL); assert.strictEqual(dirty._queue[0], KEY); })(); (function testCallback() { var KEY = 'example', VAL = {}, CB = function() {}; gently.expect(dirty, '_maybeFlush'); dirty.set(KEY, VAL, CB); assert.strictEqual(dirty._queue[1][0], KEY); assert.strictEqual(dirty._queue[1][1], CB); })(); (function testUndefinedActsAsRemove() { var KEY = 'example', VAL = undefined; gently.expect(dirty, '_maybeFlush'); dirty.set(KEY, VAL); assert.ok(!(KEY in dirty._docs)); })(); }); test(function _maybeFlush() { (function testNothingToFlush() { gently.expect(dirty, '_flush', 0); dirty._maybeFlush(); })(); (function testFlush() { dirty.flushing = false; dirty.path = '/foo/bar'; dirty._queue = [1]; gently.expect(dirty, '_flush'); dirty._maybeFlush(); })(); (function testOneFlushAtATime() { dirty.flushing = true; gently.expect(dirty, '_flush', 0); dirty._maybeFlush(); })(); (function testNoFlushingWithoutPath() { dirty.flushing = false; dirty.path = null; gently.expect(dirty, '_flush', 0); dirty._maybeFlush(); })(); (function testNoFlushingWithoutQueue() { dirty.flushing = false; dirty.path = '/foo/bar'; dirty._queue = []; gently.expect(dirty, '_flush', 0); dirty._maybeFlush(); })(); }); test(function _flush() { var WRITE_STREAM = dirty._writeStream = {}, CB; var ERR = new Error('oh oh'); gently.expect(WRITE_STREAM, 'write', function (str, cb) { assert.strictEqual(dirty.flushing, true); assert.equal( str, JSON.stringify({key: 'foo', val: 1})+'\n'+ JSON.stringify({key: 'bar', val: 2})+'\n' ); cb(ERR); }); var BAR_CB = gently.expect(function writeCb(err) { assert.strictEqual(err, ERR); }); var ERR2 = new Error('oh oh'); gently.expect(WRITE_STREAM, 'write', function (str, cb) { assert.equal(str, JSON.stringify({key: 'test', val: 3})+'\n'); cb(ERR2); }); gently.expect(dirty, 'emit', function (event, err) { assert.strictEqual(event, 'error'); assert.strictEqual(err, ERR2); }); dirty.writeBundle = 2; dirty._docs = {foo: 1, bar: 2, test: 3}; dirty._queue = ['foo', ['bar', BAR_CB], 'test']; dirty._flush(); assert.deepEqual(dirty._queue, []); }); test(function rm() { var KEY = 'foo', CB = function() {}; gently.expect(dirty, 'set', function (key, val, cb) { assert.strictEqual(key, KEY); assert.strictEqual(val, undefined); assert.strictEqual(cb, CB); }); dirty.rm(KEY, CB); }); test(function forEach() { for (var i = 1; i <= 4; i++) { dirty.set(i, {}); }; var i = 0; dirty.forEach(function(key, doc) { i++; assert.equal(key, i); assert.strictEqual(doc, dirty._docs[i]); if (i == 3) { return false; } }); assert.equal(i, 3); }); felixge-node-dirty-376f4b7/test/system/000077500000000000000000000000001171360141100200535ustar00rootroot00000000000000felixge-node-dirty-376f4b7/test/system/test-flush.js000066400000000000000000000004461171360141100225130ustar00rootroot00000000000000require('../common'); var DB_FILE = TEST_TMP+'/flush.dirty'; db = require('dirty')(DB_FILE), fs = require('fs'); db.set('foo', 'bar'); db.on('drain', function() { assert.strictEqual( fs.readFileSync(DB_FILE, 'utf-8'), JSON.stringify({key: 'foo', 'val': 'bar'})+'\n' ); }); felixge-node-dirty-376f4b7/test/system/test-for-each.js000066400000000000000000000004211171360141100230470ustar00rootroot00000000000000require('../common'); var db = require('dirty')(); db.set(1, {test: 'foo'}); db.set(2, {test: 'bar'}); db.set(3, {test: 'foobar'}); var i = 0; db.forEach(function(key, doc) { i++; assert.equal(key, i); assert.strictEqual(doc, db.get(key)); }); assert.equal(i, 3); felixge-node-dirty-376f4b7/test/system/test-load.js000066400000000000000000000011671171360141100223120ustar00rootroot00000000000000require('../common'); var DB_FILE = TEST_TMP+'/load.dirty'; db = require('dirty')(DB_FILE), fs = require('fs'), loaded = false; db.set(1, 'A'); db.set(2, 'B'); db.set(3, 'C'); db.rm(3); db.on('drain', function() { var db2 = require('dirty')(DB_FILE); db2.on('load', function(length) { loaded = true; assert.equal(length, 2); assert.strictEqual(db2.get(1), 'A'); assert.strictEqual(db2.get(2), 'B'); assert.strictEqual(db2.get(3), undefined); assert.strictEqual(db2._keys.length, 2); assert.ok(!('3' in db2._docs)); }); }); process.on('exit', function() { assert.ok(loaded); }); felixge-node-dirty-376f4b7/test/system/test-size.js000066400000000000000000000002521171360141100223370ustar00rootroot00000000000000require('../common'); var db = require(global.ROOT_LIB)(); db.set(1, {test: 'foo'}); db.set(2, {test: 'bar'}); db.set(3, {test: 'foobar'}); assert.equal(db.size(), 3); felixge-node-dirty-376f4b7/test/tmp/000077500000000000000000000000001171360141100173275ustar00rootroot00000000000000felixge-node-dirty-376f4b7/test/tmp/.empty000066400000000000000000000000001171360141100204540ustar00rootroot00000000000000