pax_global_header00006660000000000000000000000064134154125410014512gustar00rootroot0000000000000052 comment=bccc3652145f9ceaaf1035ff1258580ca39d465d memory-pager-1.5.0/000077500000000000000000000000001341541254100141215ustar00rootroot00000000000000memory-pager-1.5.0/.gitignore000066400000000000000000000000151341541254100161050ustar00rootroot00000000000000node_modules memory-pager-1.5.0/.travis.yml000066400000000000000000000000531341541254100162300ustar00rootroot00000000000000language: node_js node_js: - '4' - '6' memory-pager-1.5.0/LICENSE000066400000000000000000000020671341541254100151330ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2017 Mathias Buus Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. memory-pager-1.5.0/README.md000066400000000000000000000023331341541254100154010ustar00rootroot00000000000000# memory-pager Access memory using small fixed sized buffers instead of allocating a huge buffer. Useful if you are implementing sparse data structures (such as large bitfield). ![travis](https://travis-ci.org/mafintosh/memory-pager.svg?branch=master) ``` npm install memory-pager ``` ## Usage ``` js var pager = require('paged-memory') var pages = pager(1024) // use 1kb per page var page = pages.get(10) // get page #10 console.log(page.offset) // 10240 console.log(page.buffer) // a blank 1kb buffer ``` ## API #### `var pages = pager(pageSize)` Create a new pager. `pageSize` defaults to `1024`. #### `var page = pages.get(pageNumber, [noAllocate])` Get a page. The page will be allocated at first access. Optionally you can set the `noAllocate` flag which will make the method return undefined if no page has been allocated already A page looks like this ``` js { offset: byteOffset, buffer: bufferWithPageSize } ``` #### `pages.set(pageNumber, buffer)` Explicitly set the buffer for a page. #### `pages.updated(page)` Mark a page as updated. #### `pages.lastUpdate()` Get the last page that was updated. #### `var buf = pages.toBuffer()` Concat all pages allocated pages into a single buffer ## License MIT memory-pager-1.5.0/index.js000066400000000000000000000072231341541254100155720ustar00rootroot00000000000000module.exports = Pager function Pager (pageSize, opts) { if (!(this instanceof Pager)) return new Pager(pageSize, opts) this.length = 0 this.updates = [] this.path = new Uint16Array(4) this.pages = new Array(32768) this.maxPages = this.pages.length this.level = 0 this.pageSize = pageSize || 1024 this.deduplicate = opts ? opts.deduplicate : null this.zeros = this.deduplicate ? alloc(this.deduplicate.length) : null } Pager.prototype.updated = function (page) { while (this.deduplicate && page.buffer[page.deduplicate] === this.deduplicate[page.deduplicate]) { page.deduplicate++ if (page.deduplicate === this.deduplicate.length) { page.deduplicate = 0 if (page.buffer.equals && page.buffer.equals(this.deduplicate)) page.buffer = this.deduplicate break } } if (page.updated || !this.updates) return page.updated = true this.updates.push(page) } Pager.prototype.lastUpdate = function () { if (!this.updates || !this.updates.length) return null var page = this.updates.pop() page.updated = false return page } Pager.prototype._array = function (i, noAllocate) { if (i >= this.maxPages) { if (noAllocate) return grow(this, i) } factor(i, this.path) var arr = this.pages for (var j = this.level; j > 0; j--) { var p = this.path[j] var next = arr[p] if (!next) { if (noAllocate) return next = arr[p] = new Array(32768) } arr = next } return arr } Pager.prototype.get = function (i, noAllocate) { var arr = this._array(i, noAllocate) var first = this.path[0] var page = arr && arr[first] if (!page && !noAllocate) { page = arr[first] = new Page(i, alloc(this.pageSize)) if (i >= this.length) this.length = i + 1 } if (page && page.buffer === this.deduplicate && this.deduplicate && !noAllocate) { page.buffer = copy(page.buffer) page.deduplicate = 0 } return page } Pager.prototype.set = function (i, buf) { var arr = this._array(i, false) var first = this.path[0] if (i >= this.length) this.length = i + 1 if (!buf || (this.zeros && buf.equals && buf.equals(this.zeros))) { arr[first] = undefined return } if (this.deduplicate && buf.equals && buf.equals(this.deduplicate)) { buf = this.deduplicate } var page = arr[first] var b = truncate(buf, this.pageSize) if (page) page.buffer = b else arr[first] = new Page(i, b) } Pager.prototype.toBuffer = function () { var list = new Array(this.length) var empty = alloc(this.pageSize) var ptr = 0 while (ptr < list.length) { var arr = this._array(ptr, true) for (var i = 0; i < 32768 && ptr < list.length; i++) { list[ptr++] = (arr && arr[i]) ? arr[i].buffer : empty } } return Buffer.concat(list) } function grow (pager, index) { while (pager.maxPages < index) { var old = pager.pages pager.pages = new Array(32768) pager.pages[0] = old pager.level++ pager.maxPages *= 32768 } } function truncate (buf, len) { if (buf.length === len) return buf if (buf.length > len) return buf.slice(0, len) var cpy = alloc(len) buf.copy(cpy) return cpy } function alloc (size) { if (Buffer.alloc) return Buffer.alloc(size) var buf = new Buffer(size) buf.fill(0) return buf } function copy (buf) { var cpy = Buffer.allocUnsafe ? Buffer.allocUnsafe(buf.length) : new Buffer(buf.length) buf.copy(cpy) return cpy } function Page (i, buf) { this.offset = i * buf.length this.buffer = buf this.updated = false this.deduplicate = 0 } function factor (n, out) { n = (n - (out[0] = (n & 32767))) / 32768 n = (n - (out[1] = (n & 32767))) / 32768 out[3] = ((n - (out[2] = (n & 32767))) / 32768) & 32767 } memory-pager-1.5.0/package.json000066400000000000000000000011231341541254100164040ustar00rootroot00000000000000{ "name": "memory-pager", "version": "1.5.0", "description": "Access memory using small fixed sized buffers", "main": "index.js", "dependencies": {}, "devDependencies": { "standard": "^9.0.0", "tape": "^4.6.3" }, "scripts": { "test": "standard && tape test.js" }, "repository": { "type": "git", "url": "https://github.com/mafintosh/memory-pager.git" }, "author": "Mathias Buus (@mafintosh)", "license": "MIT", "bugs": { "url": "https://github.com/mafintosh/memory-pager/issues" }, "homepage": "https://github.com/mafintosh/memory-pager" } memory-pager-1.5.0/test.js000066400000000000000000000026441341541254100154440ustar00rootroot00000000000000var tape = require('tape') var pager = require('./') tape('get page', function (t) { var pages = pager(1024) var page = pages.get(0) t.same(page.offset, 0) t.same(page.buffer, Buffer.alloc(1024)) t.end() }) tape('get page twice', function (t) { var pages = pager(1024) t.same(pages.length, 0) var page = pages.get(0) t.same(page.offset, 0) t.same(page.buffer, Buffer.alloc(1024)) t.same(pages.length, 1) var other = pages.get(0) t.same(other, page) t.end() }) tape('get no mutable page', function (t) { var pages = pager(1024) t.ok(!pages.get(141, true)) t.ok(pages.get(141)) t.ok(pages.get(141, true)) t.end() }) tape('get far out page', function (t) { var pages = pager(1024) var page = pages.get(1000000) t.same(page.offset, 1000000 * 1024) t.same(page.buffer, Buffer.alloc(1024)) t.same(pages.length, 1000000 + 1) var other = pages.get(1) t.same(other.offset, 1024) t.same(other.buffer, Buffer.alloc(1024)) t.same(pages.length, 1000000 + 1) t.ok(other !== page) t.end() }) tape('updates', function (t) { var pages = pager(1024) t.same(pages.lastUpdate(), null) var page = pages.get(10) page.buffer[42] = 1 pages.updated(page) t.same(pages.lastUpdate(), page) t.same(pages.lastUpdate(), null) page.buffer[42] = 2 pages.updated(page) pages.updated(page) t.same(pages.lastUpdate(), page) t.same(pages.lastUpdate(), null) t.end() })