package/package.json000644 000765 000024 0000001445 13113250751013017 0ustar00000000 000000 { "name": "node-dir", "version": "0.1.17", "description": "asynchronous file and directory operations for Node.js", "main": "index", "homepage": "https://github.com/fshost", "repository": "https://github.com/fshost/node-dir", "author": { "name": "Nathan Cartwright", "email": "fshost@yahoo.com", "url": "https://github.com/fshost" }, "directories": { "lib": "lib" }, "scripts": { "test": "./node_modules/.bin/mocha --reporter spec" }, "engines": { "node": ">= 0.10.5" }, "license": "MIT", "keywords": [ "node-dir", "directory", "dir", "subdir", "file", "asynchronous", "Node.js", "fs" ], "dependencies": { "minimatch": "^3.0.2" }, "devDependencies": { "mocha": "~1.13.0", "should": "~2.0.2" } } package/.npmignore000644 000765 000024 0000000026 13113250751012522 0ustar00000000 000000 .git* .travis.yml testpackage/README.md000644 000765 000024 0000017752 13113250751012020 0ustar00000000 000000 [![Build Status](https://secure.travis-ci.org/fshost/node-dir.svg)](http://travis-ci.org/fshost/node-dir) # node-dir A lightweight Node.js module with methods for some common directory and file operations, including asynchronous, non-blocking methods for recursively getting an array of files, subdirectories, or both, and methods for recursively, sequentially reading and processing the contents of files in a directory and its subdirectories, with several options available for added flexibility if needed. ### Table of Contents - [installation](#installation) - [usage](#usage) - [methods](#methods) - [readFiles( dir, options, fileCallback, finishedCallback)](#readfiles-dir-options-filecallback-finishedcallback) - [readFilesStream( dir, options, streamCallback, finishedCallback)](#readfilesstream-dir-options-streamcallback-finishedcallback) - [readFilesStream examples](#readfilesstream-examples) - [files( dir, callback )](#files-dir-callback) - [files( dir, {sync:true} )](#files-dir-synctrue) - [promiseFiles( dir, callback )](#promisefiles-dir-callback) - [subdirs( dir, callback )](#subdirs-dir-callback) - [paths(dir, [combine], callback )](#pathsdir-combine-callback) - [API Docs](#api-docs) - [files(dir, type, callback, options)](#filesdir-type-callback-options) - [License](#license) #### installation npm install node-dir ### usage #### methods For the sake of brevity, assume that the following line of code precedes all of the examples. ```javascript var dir = require('node-dir'); ``` #### readFiles( dir, [options], fileCallback, [finishedCallback] ) #### readFilesStream( dir, [options], streamCallback, [finishedCallback] ) Sequentially read the content of each file in a directory, passing the contents to a callback, optionally calling a finished callback when complete. The options and finishedCallback arguments are not required. Valid options are: - encoding: file encoding (defaults to 'utf8') - exclude: a regex pattern or array to specify filenames to ignore - excludeDir: a regex pattern or array to specify directories to ignore - match: a regex pattern or array to specify filenames to operate on - matchDir: a regex pattern or array to specify directories to recurse - recursive: whether to recurse subdirectories when reading files (defaults to true) - reverse: sort files in each directory in descending order - shortName: whether to aggregate only the base filename rather than the full filepath - sort: sort files in each directory in ascending order (defaults to true) - doneOnErr: control if done function called on error (defaults to true) A reverse sort can also be achieved by setting the sort option to 'reverse', 'desc', or 'descending' string value. #### readFilesStream examples ```javascript // display contents of files in this script's directory dir.readFiles(__dirname, function(err, content, next) { if (err) throw err; console.log('content:', content); next(); }, function(err, files){ if (err) throw err; console.log('finished reading files:', files); }); // display contents of huge files in this script's directory dir.readFilesStream(__dirname, function(err, stream, next) { if (err) throw err; var content = ''; stream.on('data',function(buffer) { content += buffer.toString(); }); stream.on('end',function() { console.log('content:', content); next(); }); }, function(err, files){ if (err) throw err; console.log('finished reading files:', files); }); // match only filenames with a .txt extension and that don't start with a `.´ dir.readFiles(__dirname, { match: /.txt$/, exclude: /^\./ }, function(err, content, next) { if (err) throw err; console.log('content:', content); next(); }, function(err, files){ if (err) throw err; console.log('finished reading files:',files); }); // exclude an array of subdirectory names dir.readFiles(__dirname, { exclude: ['node_modules', 'test'] }, function(err, content, next) { if (err) throw err; console.log('content:', content); next(); }, function(err, files){ if (err) throw err; console.log('finished reading files:',files); }); // the callback for each file can optionally have a filename argument as its 3rd parameter // and the finishedCallback argument is optional, e.g. dir.readFiles(__dirname, function(err, content, filename, next) { console.log('processing content of file', filename); next(); }); ``` #### files( dir, callback ) Asynchronously iterate the files of a directory and its subdirectories and pass an array of file paths to a callback. ```javascript dir.files(__dirname, function(err, files) { if (err) throw err; console.log(files); }); ``` #### files( dir, {sync:true} ) Synchronously iterate the files of a directory and its subdirectories and pass an array of file paths to a callback. ```javascript var files = dir.files(__dirname, {sync:true}); console.log(files); ``` #### promiseFiles( dir, callback ) Asynchronously iterate the files of a directory and its subdirectories and pass an array of file paths to a callback. ```javascript dir.promiseFiles(__dirname) .then((files)=>{ console.log(files); }) .catch(e=>console.error(e)) ``` Note that for the files and subdirs the object returned is an array, and thus all of the standard array methods are available for use in your callback for operations like filters or sorting. Some quick examples: ```javascript dir.files(__dirname, function(err, files) { if (err) throw err; // sort ascending files.sort(); // sort descending files.reverse(); // include only certain filenames files = files.filter(function (file) { return ['allowed', 'file', 'names'].indexOf(file) > -1; }); // exclude some filenames files = files.filter(function (file) { return ['exclude', 'these', 'files'].indexOf(file) === -1; }); }); ``` Also note that if you need to work with the contents of the files asynchronously, please use the readFiles method. The files and subdirs methods are for getting a list of the files or subdirs in a directory as an array. #### subdirs( dir, callback ) Asynchronously iterate the subdirectories of a directory and its subdirectories and pass an array of directory paths to a callback. ```javascript dir.subdirs(__dirname, function(err, subdirs) { if (err) throw err; console.log(subdirs); }); ``` #### paths(dir, [combine], callback ) Asynchronously iterate the subdirectories of a directory and its subdirectories and pass an array of both file and directory paths to a callback. Separated into two distinct arrays (paths.files and paths.dirs) ```javascript dir.paths(__dirname, function(err, paths) { if (err) throw err; console.log('files:\n',paths.files); console.log('subdirs:\n', paths.dirs); }); ``` Combined in a single array (convenience method for concatenation of the above) ```javascript dir.paths(__dirname, true, function(err, paths) { if (err) throw err; console.log('paths:\n',paths); }); ``` ## API Docs ### files(dir, type, callback, options) - **dir** - directory path to read - **type**='file' - 'file' returns only file listings - 'dir' returns only directory listings - 'all' returns {dirs:[], files:[]} - 'combine' returns [] - **callback** - - **options** - **sync**=false - results are returned inline and no callbacks are used - **shortName**=false - instead of fullpath file names, just get the names - **recursive**=true - traverse through all children of given path ## License MIT licensed (See LICENSE.txt) package/index.js000644 000765 000024 0000000272 13113250751012173 0ustar00000000 000000 var dirpaths = require('./lib/paths'); Object.assign(exports, dirpaths) exports.readFiles = require('./lib/readfiles'); exports.readFilesStream = require('./lib/readfilesstream'); package/LICENSE.txt000644 000765 000024 0000002112 13113250751012344 0ustar00000000 000000 (The MIT License) Copyright (c) 2012 Nathan Cartwright Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.package/lib/paths.js000644 000765 000024 0000014406 13113250751012755 0ustar00000000 000000 var fs = require('fs'), path = require('path'); exports.promiseFiles = function promiseFiles(dir, type, options){ type = type || 'file' var processor = function(res,rej){ var cb = function(err,data){ if(err)return rej(err) res(data) } exports.files(dir,type,cb,options) } return new Promise(processor) } /** * find all files or subdirs (recursive) and pass to callback fn * * @param {string} dir directory in which to recurse files or subdirs * @param {string} type type of dir entry to recurse ('file', 'dir', or 'all', defaults to 'file') * @param {function(error, )} callback fn to call when done * @example * dir.files(__dirname, function(err, files) { * if (err) throw err; * console.log('files:', files); * }); */ exports.files = function files(dir, type, callback, options) { var ofType = typeof type if(ofType == 'object'){ options = options || type type = 'file' callback = function(){} }else if (ofType !== 'string') { //ignoreType = callback; callback = type; type = 'file'; } options = options || {} var pending, results = { files: [], dirs: [] }; var done = function() { if(type==='combine'){ results = results.files.concat(results.dirs) } else if (!type || options.ignoreType || ['all','combine'].indexOf(type)>=0) { results = results } else { results = results[type + 's'] } if(options.sync)return; callback(null, results); }; var getStatHandler = function(statPath, name, lstatCalled) { return function(err, stat) { if (err) { if (!lstatCalled) { return fs.lstat(statPath, getStatHandler(statPath, name, true)); } return callback(err); } var pushVal = options.shortName ? name : statPath if (stat && stat.isDirectory() && stat.mode !== 17115) { if (type !== 'file') { results.dirs.push(pushVal); } if (options.recursive==null || options.recursive) { var subloop = function(err, res) { if (err){ return callback(err) } if(type === 'combine'){ results.files = results.files.concat(res); }else if (type === 'all') { results.files = results.files.concat(res.files); results.dirs = results.dirs.concat(res.dirs); } else if (type === 'file') { results.files = results.files.concat(res.files); } else { results.dirs = results.dirs.concat(res.dirs); } if (!--pending){ done(); } } var newOptions = Object.assign({}, options) newOptions.ignoreType = true var moreResults = files(statPath, type, subloop, newOptions); if(options.sync){ subloop(null, moreResults) } }else if (!--pending){ done() } } else { if (type !== 'dir') { results.files.push(pushVal); } // should be the last statement in statHandler if (!--pending){ done() } } } } var bufdir = Buffer.from(dir); const onDirRead = function(err, list) { if (err) return callback(err); pending = list.length; if (!pending) return done(); for (var file, i = 0, l = list.length; i < l; i++) { var fname = list[i].toString(); file = path.join(dir, fname); var buffile = Buffer.concat([bufdir, Buffer.from(path.sep), list[i]]); if(options.sync){ var res = fs.statSync(buffile); getStatHandler(file,fname)(null, res) }else{ fs.stat(buffile, getStatHandler(file,fname)); } } return results } const onStat = function(err, stat) { if (err) return callback(err); if (stat && stat.mode === 17115) return done(); if(options.sync){ const list = fs.readdirSync(bufdir, {encoding: 'buffer'}) return onDirRead(null, list) }else{ fs.readdir(bufdir, {encoding: 'buffer'}, onDirRead) } } if(options.sync){ const stat = fs.statSync(bufdir); return onStat(null, stat) }else{ fs.stat(bufdir, onStat); } }; /** * find all files and subdirs in a directory (recursive) and pass them to callback fn * * @param {string} dir directory in which to recurse files or subdirs * @param {boolean} combine whether to combine both subdirs and filepaths into one array (default false) * @param {function(error, Object.<, Array.>)} callback fn to call when done * @example * dir.paths(__dirname, function (err, paths) { * if (err) throw err; * console.log('files:', paths.files); * console.log('subdirs:', paths.dirs); * }); * dir.paths(__dirname, true, function (err, paths) { * if (err) throw err; * console.log('paths:', paths); * }); */ exports.paths = function paths(dir, combine, callback) { var type; if (typeof combine === 'function') { callback = combine; combine = false; } exports.files(dir, 'all', function(err, results) { if (err) return callback(err); if (combine) { callback(null, results.files.concat(results.dirs)); } else { callback(null, results); } }); }; /** * find all subdirs (recursive) of a directory and pass them to callback fn * * @param {string} dir directory in which to find subdirs * @param {string} type type of dir entry to recurse ('file' or 'dir', defaults to 'file') * @param {function(error, )} callback fn to call when done * @example * dir.subdirs(__dirname, function (err, paths) { * if (err) throw err; * console.log('files:', paths.files); * console.log('subdirs:', paths.dirs); * }); */ exports.subdirs = function subdirs(dir, callback, type, options) { options = options || {} const iCallback = function(err, subdirs) { if (err) return callback(err); if(type=='combine'){ subdirs = subdirs.files.concat(subdirs.dirs) } if(options.sync)return subdirs callback(null, subdirs); } const res = exports.files(dir, 'dir', iCallback, options) if(options && options.sync){ return iCallback(null,res) } }; package/lib/readfiles.js000644 000765 000024 0000011024 13113250751013565 0ustar00000000 000000 var fs = require('fs'), path = require('path'); /** * merge two objects by extending target object with source object * @param target object to merge * @param source object to merge * @param {Boolean} [modify] whether to modify the target * @returns {Object} extended object */ function extend(target, source, modify) { var result = target ? modify ? target : extend({}, target, true) : {}; if (!source) return result; for (var key in source) { if (source.hasOwnProperty(key) && source[key] !== undefined) { result[key] = source[key]; } } return result; } /** * determine if a string is contained within an array or matches a regular expression * @param {String} str string to match * @param {Array|Regex} match array or regular expression to match against * @returns {Boolean} whether there is a match */ function matches(str, match) { if (Array.isArray(match)) return match.indexOf(str) > -1; return match.test(str); } /** * read files and call a function with the contents of each file * @param {String} dir path of dir containing the files to be read * @param {String} encoding file encoding (default is 'utf8') * @param {Object} options options hash for encoding, recursive, and match/exclude * @param {Function(error, string)} callback callback for each files content * @param {Function(error)} complete fn to call when finished */ function readFiles(dir, options, callback, complete) { if (typeof options === 'function') { complete = callback; callback = options; options = {}; } if (typeof options === 'string') options = { encoding: options }; options = extend({ recursive: true, encoding: 'utf8', doneOnErr: true }, options); var files = []; var done = function(err) { if (typeof complete === 'function') { if (err) return complete(err); complete(null, files); } }; fs.readdir(dir, function(err, list) { if (err)  { if (options.doneOnErr === true) { if (err.code === 'EACCES') return done(); return done(err); } } var i = 0; if (options.reverse === true || (typeof options.sort == 'string' && (/reverse|desc/i).test(options.sort))) { list = list.reverse(); } else if (options.sort !== false) list = list.sort(); (function next() { var filename = list[i++]; if (!filename) return done(null, files); var file = path.join(dir, filename); fs.stat(file, function(err, stat) { if (err && options.doneOnErr === true) return done(err); if (stat && stat.isDirectory()) { if (options.recursive) { if (options.matchDir && !matches(filename, options.matchDir)) return next(); if (options.excludeDir && matches(filename, options.excludeDir)) return next(); readFiles(file, options, callback, function(err, sfiles) { if (err && options.doneOnErr === true) return done(err); files = files.concat(sfiles); next(); }); } else next(); } else if (stat && stat.isFile()) { if (options.match && !matches(filename, options.match)) return next(); if (options.exclude && matches(filename, options.exclude)) return next(); if (options.filter && !options.filter(filename)) return next(); if (options.shortName) files.push(filename); else files.push(file); fs.readFile(file, options.encoding, function(err, data) { if (err) { if (err.code === 'EACCES') return next(); if (options.doneOnErr === true) { return done(err); } } if (callback.length > 3) if (options.shortName) callback(null, data, filename, next); else callback(null, data, file, next); else callback(null, data, next); }); } else { next(); } }); })(); }); } module.exports = readFiles; package/lib/readfilesstream.js000644 000765 000024 0000011325 13113250751015005 0ustar00000000 000000 var fs = require('fs'), mm = require('minimatch'), path = require('path'); /** * merge two objects by extending target object with source object * @param target object to merge * @param source object to merge * @param {Boolean} [modify] whether to modify the target * @returns {Object} extended object */ function extend(target, source, modify) { var result = target ? modify ? target : extend({}, target, true) : {}; if (!source) return result; for (var key in source) { if (source.hasOwnProperty(key) && source[key] !== undefined) { result[key] = source[key]; } } return result; } /** * determine if a string is contained within an array or matches a regular expression * @param {String} str string to match * @param {Array|Regex} match array or regular expression to match against * @returns {Boolean} whether there is a match */ function matches(str, match) { if (Array.isArray(match)) { var l = match.length; for( var s=0; s < l; s++) { if ( mm(str,match[s])) { return true; } } return false; } return match.test(str); } /** * read files and call a function with the contents of each file * @param {String} dir path of dir containing the files to be read * @param {String} encoding file encoding (default is 'utf8') * @param {Object} options options hash for encoding, recursive, and match/exclude * @param {Function(error, string)} callback callback for each files content * @param {Function(error)} complete fn to call when finished */ function readFilesStream(dir, options, callback, complete) { if (typeof options === 'function') { complete = callback; callback = options; options = {}; } if (typeof options === 'string') options = { encoding: options }; options = extend({ recursive: true, encoding: 'utf8', doneOnErr: true }, options); var files = []; var done = function(err) { if (typeof complete === 'function') { if (err) return complete(err); complete(null, files); } }; fs.readdir(dir, function(err, list) { if (err)  { if (options.doneOnErr === true) { if (err.code === 'EACCES') return done(); return done(err); } } var i = 0; if (options.reverse === true || (typeof options.sort == 'string' && (/reverse|desc/i).test(options.sort))) { list = list.reverse(); } else if (options.sort !== false) list = list.sort(); (function next() { var filename = list[i++]; if (!filename) return done(null, files); var file = path.join(dir, filename); fs.stat(file, function(err, stat) { if (err && options.doneOnErr === true) return done(err); if (stat && stat.isDirectory()) { if (options.recursive) { if (options.matchDir && !matches(filename, options.matchDir)) return next(); if (options.excludeDir && matches(filename, options.excludeDir)) return next(); readFilesStream(file, options, callback, function(err, sfiles) { if (err && options.doneOnErr === true) return done(err); files = files.concat(sfiles); next(); }); } else next(); } else if (stat && stat.isFile()) { if (options.match && !matches(filename, options.match)) return next(); if (options.exclude && matches(filename, options.exclude)) return next(); if (options.filter && !options.filter(filename)) return next(); if (options.shortName) files.push(filename); else files.push(file); var stream = fs.createReadStream(file); if (options.encoding !== null) { stream.setEncoding(options.encoding); } stream.on('error',function(err) { if (options.doneOnErr === true) return done(err); next(); }); if (callback.length > 3) if (options.shortName) callback(null, stream, filename, next); else callback(null, stream, file, next); else callback(null, stream, next); } else { next(); } }); })(); }); } module.exports = readFilesStream;