pax_global_header00006660000000000000000000000064135003747450014522gustar00rootroot0000000000000052 comment=0f40e3d18ac9527a5efeca2f5a9fe25297a243dc first-chunk-stream-4.0.0/000077500000000000000000000000001350037474500152515ustar00rootroot00000000000000first-chunk-stream-4.0.0/.editorconfig000066400000000000000000000002571350037474500177320ustar00rootroot00000000000000root = true [*] indent_style = tab end_of_line = lf charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true [*.yml] indent_style = space indent_size = 2 first-chunk-stream-4.0.0/.gitattributes000066400000000000000000000000231350037474500201370ustar00rootroot00000000000000* text=auto eol=lf first-chunk-stream-4.0.0/.github/000077500000000000000000000000001350037474500166115ustar00rootroot00000000000000first-chunk-stream-4.0.0/.github/funding.yml000066400000000000000000000001331350037474500207630ustar00rootroot00000000000000github: sindresorhus open_collective: sindresorhus custom: https://sindresorhus.com/donate first-chunk-stream-4.0.0/.gitignore000066400000000000000000000000431350037474500172360ustar00rootroot00000000000000node_modules yarn.lock .nyc_output first-chunk-stream-4.0.0/.npmrc000066400000000000000000000000231350037474500163640ustar00rootroot00000000000000package-lock=false first-chunk-stream-4.0.0/.travis.yml000066400000000000000000000000651350037474500173630ustar00rootroot00000000000000language: node_js node_js: - '12' - '10' - '8' first-chunk-stream-4.0.0/index.d.ts000066400000000000000000000041431350037474500171540ustar00rootroot00000000000000import { Duplex as DuplexStream, DuplexOptions as DuplexStreamOption } from 'stream'; declare const stop: unique symbol; declare namespace FirstChunkStream { interface Options extends Readonly { /** How many bytes you want to buffer. */ readonly chunkSize: number; } type StopSymbol = typeof stop; type BufferLike = string | Buffer | Uint8Array; type TransformFunction = (chunk: Buffer, encoding: string) => Promise; } declare class FirstChunkStream extends DuplexStream { /** Buffer and transform the `n` first bytes of a stream. @param options - The options object is passed to the [`Duplex` stream](https://nodejs.org/api/stream.html#stream_class_stream_duplex) constructor allowing you to customize your stream behavior. @param transform - Async function that receives the required `options.chunkSize` bytes. Note that the buffer can have a smaller length than the required one. In that case, it will be due to the fact that the complete stream contents has a length less than the `options.chunkSize` value. You should check for this yourself if you strictly depend on the length. @example ``` import * as fs from 'fs'; import getStream = require('get-stream'); import FirstChunkStream = require('first-chunk-stream'); // unicorn.txt => unicorn rainbow const stream = fs.createReadStream('unicorn.txt') .pipe(new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { return chunk.toString(encoding).toUpperCase(); })); (async () => { const data = await getStream(stream); if (data.length < 7) { throw new Error('Couldn\'t get the minimum required first chunk length'); } console.log(data); //=> 'UNICORN rainbow' })(); ``` */ constructor( options: FirstChunkStream.Options, transform: FirstChunkStream.TransformFunction ); /** Symbol used to end the stream early. @example ``` new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { return FirstChunkStream.stop; }); ``` */ static readonly stop: FirstChunkStream.StopSymbol; } export = FirstChunkStream; first-chunk-stream-4.0.0/index.js000066400000000000000000000074771350037474500167350ustar00rootroot00000000000000'use strict'; const {Duplex: DuplexStream} = require('stream'); const stop = Symbol('FirstChunkStream.stop'); class FirstChunkStream extends DuplexStream { constructor(options, callback) { const state = { sent: false, chunks: [], size: 0 }; if (typeof options !== 'object' || options === null) { throw new TypeError('FirstChunkStream constructor requires `options` to be an object.'); } if (typeof callback !== 'function') { throw new TypeError('FirstChunkStream constructor requires a callback as its second argument.'); } if (typeof options.chunkSize !== 'number') { throw new TypeError('FirstChunkStream constructor requires `options.chunkSize` to be a number.'); } if (options.objectMode) { throw new Error('FirstChunkStream doesn\'t support `objectMode` yet.'); } super(options); // Initialize the internal state state.manager = createReadStreamBackpressureManager(this); const processCallback = (buffer, encoding, done) => { state.sent = true; (async () => { let result; try { result = await callback(buffer, encoding); } catch (error) { setImmediate(() => { this.emit('error', error); done(); }); return; } if (result === stop) { state.manager.programPush(null, undefined, done); } else if (Buffer.isBuffer(result) || (result instanceof Uint8Array) || (typeof result === 'string')) { state.manager.programPush(result, undefined, done); } else { state.manager.programPush(result.buffer, result.encoding, done); } })(); }; // Writes management this._write = (chunk, encoding, done) => { state.encoding = encoding; if (state.sent) { state.manager.programPush(chunk, state.encoding, done); } else if (chunk.length < options.chunkSize - state.size) { state.chunks.push(chunk); state.size += chunk.length; done(); } else { state.chunks.push(chunk.slice(0, options.chunkSize - state.size)); chunk = chunk.slice(options.chunkSize - state.size); state.size += state.chunks[state.chunks.length - 1].length; processCallback(Buffer.concat(state.chunks, state.size), state.encoding, () => { if (chunk.length === 0) { done(); return; } state.manager.programPush(chunk, state.encoding, done); }); } }; this.on('finish', () => { if (!state.sent) { return processCallback(Buffer.concat(state.chunks, state.size), state.encoding, () => { state.manager.programPush(null, state.encoding); }); } state.manager.programPush(null, state.encoding); }); } } // Utils to manage readable stream backpressure function createReadStreamBackpressureManager(readableStream) { const manager = { waitPush: true, programmedPushs: [], programPush(chunk, encoding, isDone = () => {}) { // Store the current write manager.programmedPushs.push([chunk, encoding, isDone]); // Need to be async to avoid nested push attempts // Programm a push attempt setImmediate(manager.attemptPush); // Let's say we're ready for a read readableStream.emit('readable'); readableStream.emit('drain'); }, attemptPush() { let nextPush; if (manager.waitPush) { if (manager.programmedPushs.length > 0) { nextPush = manager.programmedPushs.shift(); manager.waitPush = readableStream.push(nextPush[0], nextPush[1]); (nextPush[2])(); } } else { setImmediate(() => { // Need to be async to avoid nested push attempts readableStream.emit('readable'); }); } } }; function streamFilterRestoreRead() { manager.waitPush = true; // Need to be async to avoid nested push attempts setImmediate(manager.attemptPush); } // Patch the readable stream to manage reads readableStream._read = streamFilterRestoreRead; return manager; } FirstChunkStream.stop = stop; module.exports = FirstChunkStream; first-chunk-stream-4.0.0/index.test-d.ts000066400000000000000000000030251350037474500201270ustar00rootroot00000000000000/// import * as fs from 'fs'; import {Duplex} from 'stream'; import {expectType, expectError} from 'tsd'; import FirstChunkStream = require('.'); const options: FirstChunkStream.Options = {chunkSize: 1}; expectError(new FirstChunkStream({}, () => {})); const firstChunkStream = new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { expectType(chunk); expectType(encoding); return ''; }); expectType(firstChunkStream); expectType(firstChunkStream); fs.createReadStream('unicorn.txt').pipe(firstChunkStream); expectType(new FirstChunkStream({chunkSize: 7}, async () => FirstChunkStream.stop)); expectType(new FirstChunkStream({chunkSize: 7}, async () => '')); expectType(new FirstChunkStream({chunkSize: 7}, async () => Buffer.from(''))); expectType(new FirstChunkStream({chunkSize: 7}, async () => 'string')); expectType(new FirstChunkStream({chunkSize: 7}, async () => new Uint8Array(0))); expectType(new FirstChunkStream({chunkSize: 7}, async () => { return {buffer: Buffer.from('')}; })); expectType(new FirstChunkStream({chunkSize: 7}, async () => { return {buffer: new Uint8Array(0)}; })); expectType(new FirstChunkStream({chunkSize: 7}, async () => { return {buffer: 'string'}; })); expectType(new FirstChunkStream({chunkSize: 7}, async () => { return {buffer: 'string', encoding: 'utf8'}; })); first-chunk-stream-4.0.0/license000066400000000000000000000021251350037474500166160ustar00rootroot00000000000000MIT License Copyright (c) Sindre Sorhus (sindresorhus.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. first-chunk-stream-4.0.0/package.json000066400000000000000000000013241350037474500175370ustar00rootroot00000000000000{ "name": "first-chunk-stream", "version": "4.0.0", "description": "Buffer and transform the n first bytes of a stream", "license": "MIT", "repository": "sindresorhus/first-chunk-stream", "author": { "name": "Sindre Sorhus", "email": "sindresorhus@gmail.com", "url": "sindresorhus.com" }, "engines": { "node": ">=8" }, "scripts": { "test": "xo && nyc ava && tsd" }, "files": [ "index.js", "index.d.ts" ], "keywords": [ "buffer", "stream", "streams", "transform", "first", "chunk", "size", "min", "minimum", "bytes" ], "devDependencies": { "@types/node": "^12.0.8", "ava": "^2.1.0", "nyc": "^14.0.0", "streamtest": "^1.2.1", "tsd": "^0.7.3", "xo": "^0.24.0" } } first-chunk-stream-4.0.0/readme.md000066400000000000000000000050361350037474500170340ustar00rootroot00000000000000# first-chunk-stream [![Build Status](https://travis-ci.org/sindresorhus/first-chunk-stream.svg?branch=master)](https://travis-ci.org/sindresorhus/first-chunk-stream) > Buffer and transform the n first bytes of a stream ## Install ``` $ npm install first-chunk-stream ``` ## Usage ```js const fs = require('fs'); const getStream = require('get-stream'); const FirstChunkStream = require('first-chunk-stream'); // unicorn.txt => unicorn rainbow const stream = fs.createReadStream('unicorn.txt') .pipe(new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { return chunk.toString(encoding).toUpperCase(); })); (async () => { const data = await getStream(stream); if (data.length < 7) { throw new Error('Couldn\'t get the minimum required first chunk length'); } console.log(data); //=> 'UNICORN rainbow' })(); ``` ## API ### FirstChunkStream(options, transform) `FirstChunkStream` constructor. #### transform(chunk, encoding) Type: `Function` Async function that receives the required `options.chunkSize` bytes. Expected to return an buffer-like object or `string` or object of form {buffer: `Buffer`, encoding: `string`} to send to stream or `firstChunkStream.stop` to end stream right away. An error thrown from this function will be emitted as stream errors. Note that the buffer can have a smaller length than the required one. In that case, it will be due to the fact that the complete stream contents has a length less than the `options.chunkSize` value. You should check for this yourself if you strictly depend on the length. ```js new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { return chunk.toString(encoding).toUpperCase(); // Send string to stream }); new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { return chunk; // Send buffer to stream }); new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { return { buffer: chunk, encoding: encoding, }; // Send buffer with encoding to stream }); new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { return FirstChunkStream.stop; // End the stream early }); new FirstChunkStream({chunkSize: 7}, async (chunk, encoding) => { throw new Error('Unconditional error'); // Emit stream error }); ``` #### options Type: `object` The options object is passed to the [`Duplex` stream](https://nodejs.org/api/stream.html#stream_class_stream_duplex) constructor allowing you to customize your stream behavior. In addition, you can specify the following option: ###### chunkSize Type: `number` How many bytes you want to buffer. first-chunk-stream-4.0.0/test.js000066400000000000000000000202751350037474500165740ustar00rootroot00000000000000import test from 'ava'; import streamtest from 'streamtest'; import FirstChunkStream from '.'; const content = 'unicorn rainbows \ncake'; /* eslint-disable no-new */ test('fails when the options are not provided', t => { t.throws(() => { new FirstChunkStream(); }); }); test('fails when the callback is not provided', t => { t.throws(() => { new FirstChunkStream({chunkSize: 7}); }); }); test('fails when trying to use it in objectMode', t => { t.throws(() => { new FirstChunkStream({chunkSize: 7, objectMode: true}, () => {}); }); }); test('fails when firstChunk size is bad or missing', t => { t.throws(() => { new FirstChunkStream({chunkSize: 'feferf'}, () => {}); }); t.throws(() => { new FirstChunkStream({}, () => {}); }); }); /* eslint-enable no-new */ streamtest.versions.forEach(version => { test.cb( `for ${version} streams, emitting errors emits errors before first chunk is sent`, t => { t.plan(4); const stream = new FirstChunkStream( {chunkSize: 7}, async chunk => { t.pass(); return chunk; } ); stream.on('error', error => { t.is(error.message, 'Hey!'); }); stream.pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content); t.end(); }) ); stream.write(Buffer.from(content.slice(0, 3))); stream.emit('error', new Error('Hey!')); stream.write(Buffer.from(content.slice(3, 7))); stream.emit('error', new Error('Hey!')); stream.write(Buffer.from(content.slice(7))); stream.end(); } ); test.cb( `for ${version} streams, throwing errors from callback emits error`, t => { t.plan(2); const stream = new FirstChunkStream( {chunkSize: 7}, async () => { throw new Error('Ho!'); } ); stream.on('error', error => { t.is(error.message, 'Ho!'); }); stream.pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content.slice(7)); t.end(); }) ); stream.write(Buffer.from(content.slice(0, 7))); stream.write(Buffer.from(content.slice(7))); stream.end(); } ); test.cb(`for ${version} streams, requires a 0 length first chunk`, t => { t.plan(2); streamtest[version] .fromChunks([content]) .pipe( new FirstChunkStream( {chunkSize: 0}, async chunk => { t.is(chunk.toString('utf8'), ''); return Buffer.from('popop'); } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, 'popop' + content); t.end(); }) ); }); test.cb( `for ${version} streams, leaves content as is with a single oversized chunk`, t => { t.plan(2); streamtest[version] .fromChunks([content]) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return chunk; } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content); t.end(); }) ); } ); test.cb( `for ${version} streams, leaves content as is with required size chunk`, t => { t.plan(2); streamtest[version] .fromChunks([content.slice(0, 7), content.slice(7)]) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return chunk; } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content); t.end(); }) ); } ); test.cb( `for ${version} streams, leaves content as is with several small chunks`, t => { t.plan(2); streamtest[version] .fromChunks(content.split('')) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return chunk; } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content); t.end(); }) ); } ); test.cb( `for ${version} streams, leaves content as is even when consuming the stream in the callback`, t => { t.plan(2); const inputStream = streamtest[version].fromChunks(content.split('')); const firstChunkStream = inputStream.pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); firstChunkStream.pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content); t.end(); }) ); return chunk; } ) ); } ); test.cb(`for ${version} streams, works with insufficient content`, t => { t.plan(2); streamtest[version] .fromChunks(['a', 'b', 'c']) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), 'abc'); return Buffer.from('b'); } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, 'b'); t.end(); }) ); }); test.cb( `for ${version} streams, works with changing content when removing the first chunk`, t => { t.plan(2); streamtest[version] .fromChunks([content]) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return Buffer.alloc(0); } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content.slice(7)); t.end(); }) ); } ); test.cb( `for ${version} streams, works with string and encoding`, t => { t.plan(2); streamtest[version] .fromChunks([content]) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return {buffer: chunk.toString('utf8'), encoding: 'utf8'}; } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content); t.end(); }) ); } ); test.cb( `for ${version} streams, works with stop`, t => { t.plan(2); streamtest[version] .fromChunks([content]) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return FirstChunkStream.stop; } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, ''); t.end(); }) ); } ); test.cb( `for ${version} streams, works with changing content when replacing per a larger chunk`, t => { t.plan(2); streamtest[version] .fromChunks([content.slice(0, 7), content.slice(7)]) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return Buffer.concat([chunk, Buffer.from('plop')]); } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, content.slice(0, 7) + 'plop' + content.slice(7)); t.end(); }) ); } ); test.cb( `for ${version} streams, works with changing content when replacing per a smaller chunk`, t => { t.plan(2); streamtest[version] .fromChunks(content.split('')) .pipe( new FirstChunkStream( {chunkSize: 7}, async chunk => { t.is(chunk.toString('utf8'), content.slice(0, 7)); return Buffer.from('plop'); } ) ) .pipe( streamtest[version].toText((error, text) => { if (error) { t.end(error); return; } t.is(text, 'plop' + content.slice(7)); t.end(); }) ); } ); });