package/0000775000175000017500000000000011740371016011413 5ustar daviddavidpackage/Makefile0000644000175000017500000000026611716563260013064 0ustar daviddavid# # Run all tests # test: @mocha -R spec lint: ./node_modules/.bin/jshint lib/node-get/*.js doc: ./node_modules/.bin/docco lib/node-get/*.js bin/node-get-file.js .PHONY: test package/.travis.yml0000644000175000017500000000005311725656551013535 0ustar daviddavidlanguage: node_js node_js: - 0.4 - 0.6 package/.npmignore0000644000175000017500000000000611716556317013420 0ustar daviddavidtest/ package/CHANGELOG.md0000644000175000017500000000233511737335002013226 0ustar daviddavid## Changelog ### 1.1.4 * Retain node v0.4.x compatibility. ### 1.1.3 * Now using 10 second timeout - tests using mocha ### 1.1.2 * Better error handling around invalid URLs ### 1.1.1 * Node 0.6.3 compatibility without warnings ### 1.1.0 * Returns Get instance as last parameter of `toDisk`, which assists with filetype-guessing ### 1.0.0 * Switched from deprecated `createClient()` API to new `http.request` API from node. * Stronger support for HTTPS * No longer supports node versions below 0.3.6 ### 0.4.0 * Added `asBuffer()` method * Streamlined `asDisk` to use node's native `.pipe()` function * Added `encoding` option to constructor ### 0.4.0 * `.asBuffer()` added * `get()` can now be used without `new` ### 0.3.0 * `get` now supports HTTP SOCKS proxies by setting `HTTP_PROXY` in `ENV` ### 0.2.0 * `node-get` is now `get`. ### 0.1.1 * [laplatrem](https://github.com/leplatrem): Fixed HTTPS support ### 0.1.0 * `max_redirs`, `headers` options in node-get constructor * The API changes in 0.1.x - Get should never be expected to throw an exception. * Handling of invalid URLs on redirect. * Handling of file-level errors. ### 0.0.3 * Handling of DNS-level exceptions. ### 0.0.2 * Enhanced URL validation. package/lib/0000755000175000017500000000000011737337551012173 5ustar daviddavidpackage/lib/node-get/0000755000175000017500000000000011737337551013675 5ustar daviddavidpackage/lib/node-get/index.js0000644000175000017500000000007011716556317015337 0ustar daviddavidvar get = require('./node-get'); module.exports = get; package/lib/node-get/encodings.js0000644000175000017500000000056511716556317016212 0ustar daviddavid// Preset encodings for file formats. module.exports = { 'ext': { '.tif': 'binary', '.tiff': 'binary', '.geotiff': 'binary', '.zip': 'binary', '.sqlite': 'binary', '.png': 'binary', '.gif': 'binary', '.jpg': 'binary', '.mp3': 'binary', '.ico': 'binary', '.jpeg': 'binary' } }; package/lib/node-get/node-get.js0000644000175000017500000002437311737337301015737 0ustar daviddavid// node.js libraries var http = require('http'), https = require('https'), util = require('util'), fs = require('fs'), events = require('events'), Buffer = require('buffer').Buffer, url = require('url'), path = require('path'); // Local node-get libraries var encodings = require('./encodings'); var default_headers = { 'Accept-Encoding': 'none', 'Connection': 'close', 'User-Agent': 'curl' }; // Get a Get object. Takes an argument, that is either // a plain string representing the URI, or an object: // // { // uri: "string of uri", // required // headers: {} // optional, default in source // max_redirs: 5 // optional, default 10 // no_proxy: true // prevent automatic proxy usage when HTTP_PROXY is set. // } function Get(options) { // Allow calling without new keyword. if (!(this instanceof Get)) { return new Get(options); } if (typeof options == 'string') { this.uri = options; this.headers = default_headers; if (process.env.HTTP_PROXY) { this.proxy = url.parse(process.env.HTTP_PROXY); } else { this.proxy = {}; } } else { if (!options.uri) { throw Error('uri option required in get constructor'); } this.uri = options.uri; this.max_redirs = options.max_redirs || 10; this.max_length = options.max_length || 0; this.encoding = options.encoding; this.headers = options.headers || default_headers; this.timeout = 'timeout' in options ? options.timeout : 10000; if (!this.no_proxy && process.env.HTTP_PROXY) { this.proxy = url.parse(process.env.HTTP_PROXY); } else { this.proxy = {}; } } } util.inherits(Get, events.EventEmitter); // Create a HTTP request. Just sanity-checks URLs and // chooses an HTTP or HTTPS request. // // - @return {http.ClientRequest} Get.prototype.request = function(callback) { // TODO: handle non http/https protocols this.uri_o = url.parse(this.uri); // Validate the URI at this step so that invalid // redirects are also caught. if (!(this.uri_o.protocol && (this.uri_o.protocol == 'http:' || this.uri_o.protocol == 'https:') && this.uri_o.hostname)) { return callback.call(this, null, new Error('Invalid URL')); } // TODO: should pronode-getxies support HTTPS? if (this.uri_o.protocol == 'https:') { return https.request({ host: this.uri_o.hostname, port: 443, path: this.proxy.hostname ? this.uri : ((this.uri_o.pathname || '') + (this.uri_o.search || '') + (this.uri_o.hash || '')) || '/' }, callback); } else { return http.request({ port: this.proxy.port || this.uri_o.port || 80, host: this.proxy.hostname || this.uri_o.hostname, path: this.proxy.hostname ? this.uri : ((this.uri_o.pathname || '') + (this.uri_o.search || '') + (this.uri_o.hash || '')) || '/' }, callback); } }; // Innermost API function of Get // // - @param {Function} callback // - @param {Number} times number of times re-called. Get.prototype.perform = function(callback, times) { if (times > this.max_redirs) { return callback(new Error('Redirect limit of ' + this.max_redirs + ' reached')); } times = times || 1; var clientrequest = this.request(function handleClientRequest(response, err) { if (err) return callback.call(this, err, null); if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) { // Redirection // ----------- // Servers can send a full redirect location // or a short form, like a hyperlink. Handle both. if (url.parse(response.headers.location).protocol) { this.uri = response.headers.location; } else { this.uri = url.resolve(this.uri, response.headers.location); } this.perform(callback, times + 1); return; } else if (response.statusCode >= 400) { // failure var err = new Error('Server returned HTTP ' + response.statusCode); err.status = response.statusCode; return callback.call(this, err, response); } else { // success return callback.call(this, null, response); } }.bind(this)); // The client can fail if the url is invalid if (clientrequest) { // Ensure the callback is only called once in error cases. // Timeouts can trigger both error and timeout callbacks. var error = 0; // Handle DNS-level errors, like ECONNREFUSED clientrequest.on('error', function(err) { if (++error > 1) return; return callback.call(this, err); }.bind(this)); // Enforce a timeout of 10 seconds. // Add a no-op version of setTimeout for node <= 0.4.x. clientrequest.setTimeout = clientrequest.setTimeout || function() {}; clientrequest.setTimeout(this.timeout, function() { clientrequest.connection.end(); if (++error > 1) return; return callback.call(this, new Error('Timed out after ' + this.timeout + 'ms')); }.bind(this)); // TODO: fix when/if gzip is supported. // If a proxy is defined, ask for the full requested URL, // otherwise construct the URL without a hostname and protocol. clientrequest.end(); } }; Get.prototype.guessResponseExtension = function(response) { if (response.headers['content-disposition']) { var match = response.headers['content-disposition'].match(/filename=\"([^"]+)\"/); if (match) { var ext = path.extname(match[1]); if (ext) { return ext; } } } return false; }; // Stream a file to disk // --------------------- // - @param {String} filename. // - @param {Function} callback. Get.prototype.toDisk = function(filename, callback) { // TODO: catch all errors this.perform(function(err, response) { if (err) return callback(err); // Don't set an encoding. Using an encoding messes up binary files. // Pump contents from the response stream into a new writestream. var file = fs.createWriteStream(filename); file.on('error', callback); file.on('close', function() { return callback(null, filename, response, this); }.bind(this)); response.pipe(file); }); }; // Get the contents of a URL as a string // // - @param {Function} callback. Get.prototype.asString = function(callback) { var max_length = this.max_length; var payload = 0; // TODO: catch all errors this.perform(function pipeResponseToString(err, response) { if (err) return callback(err); switch ((response.headers['content-type'] || '').toLowerCase().split('/')[0]) { case 'binary': case 'application': case 'image': case 'video': return callback(new Error("Can't download binary file as string")); default: // TODO: respect Content-Transfer-Encoding header response.setEncoding(this.guessEncoding(this.uri)); } function returnString() { if (!callback) return; callback(null, out.join(''), response.headers); callback = null; } // Fill an array with chunks of data, // and then join it into a string before calling `callback` var out = []; response.on('data', function(chunk) { if (!callback) return; payload += chunk.length; if (max_length && payload > max_length) { response.socket.end(); callback(new Error('File exceeds maximum allowed length of ' + max_length + ' bytes')); callback = null; } else { out.push(chunk); } }); response.on('error', function(err) { if (!callback) return; callback(err); callback = null; }); response.on('end', returnString); response.on('close', returnString); }); }; // Get the contents of a URL as a buffer // // - @param {Function} callback. Get.prototype.asBuffer = function(callback) { var max_length = this.max_length; var payload = 0; this.perform(function(err, response) { if (err) return callback(err); function returnBuffer() { if (!callback) return; for (var length = 0, i = 0; i < out.length; i++) { length += out[i].length; } var result = new Buffer(length); for (var pos = 0, j = 0; j < out.length; j++) { out[j].copy(result, pos); pos += out[j].length; } callback(null, result, response.headers); callback = null; } // Fill an array with chunks of data, // and then join it into a buffer before calling `callback` var out = []; response.on('data', function(chunk) { if (!callback) return; payload += chunk.length; if (max_length && payload > max_length) { response.socket.end(); callback(new Error('File exceeds maximum allowed length of ' + max_length + ' bytes')); callback = null; } else { out.push(chunk); } }); response.on('error', function(err) { if (!callback) return; callback(err); callback = null; }); response.on('end', returnBuffer); response.on('close', returnBuffer); }); }; Get.prototype.guessEncoding = function(location) { // The 'most reliable' measure is probably the end of files, so // start off with extname. if (this.encoding) return this.encoding; var ext = path.extname(location).toLowerCase(); if (encodings.ext[ext]) return encodings.ext[ext]; }; module.exports = Get; package/package.json0000644000175000017500000000145211737337540013713 0ustar daviddavid{ "name": "get", "description": "A slightly higher-level HTTP client for node.", "version": "1.1.5", "main": "./lib/node-get/index.js", "keywords": ["http", "client", "request", "get"], "url": "http://github.com/developmentseed/node-get", "repositories": [{ "type": "git", "url": "git://github.com/developmentseed/node-get.git" }], "author": "Tom MacWright ", "contributors": [ "Konstantin Käfer " ], "licenses": [{ "type": "BSD" }], "devDependencies": { "mocha": ">= 0.11", "jshint": "0.2.x", "underscore": "1.2.x", "docco": "0.3.x" }, "engines": { "node": ">= 0.3.6" }, "scripts": { "test": "mocha -R spec" } } package/LICENSE0000644000175000017500000000271411716556317012436 0ustar daviddavidCopyright (c), Development Seed All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name "Development Seed" nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package/README.md0000644000175000017500000000421411737335002012672 0ustar daviddavid# get [![Build Status](https://secure.travis-ci.org/developmentseed/node-get.png?branch=master)](http://travis-ci.org/developmentseed/node-get) `get` is a slightly higher-level HTTP client for nodejs. ## Installation npm install get get has no dependencies. For testing, you'll need make and [expresso](https://github.com/visionmedia/expresso). For docs you'll need [docco](https://github.com/jashkenas/docco). ## Features * Redirect following. * Convenience functions for downloading and getting data as string. * Binary-extension and basic binary detection. * Configurable headers ## API Downloads are objects in `get`. ```javascript var dl = new get({ uri: 'http://google.com/' }); ``` The get constructor can also take a plain string if you don't want to give options. ```javascript var dl = new get('http://google.com/'); ``` It can also take other options. ```javascript var dl = new get({ uri: 'http://google.com/', max_redirs: 20, }); ``` Then it exposes three main methods ```javascript dl.asString(function(err, str) { console.log(str); }); ``` and ```javascript dl.toDisk('myfile.txt', function(err, filename) { console.log(err); }); ``` and finally ```javascript dl.asBuffer(function(err, data) { console.log(data); }); ``` There's also a lower-level API. ```javascript dl.perform(function(err, response) { // response is just a response object, just like // HTTP request, except handling redirects }); ``` If you give node-get an object of settings instead of a string, it accepts * `uri` - the address of the resource * `headers` - to replace its default headers with custom ones * `max_redirs` - the number of redirects to follow before returning an error * `no_proxy` - don't use a HTTP proxy, even if one is in `ENV` * `encoding` - When calling `.guessEncoding()`, `get` will use this instead of the default value ## Example ``` var get = require('get'); new get('http://google.com/').asString(function(err, data) { if (err) throw err; console.log(data); }); ``` ## TODO: * Guessing encoding wth headers * User-customizable encodings ## Authors * Tom MacWright (tmcw) * Konstantin Kaefer (kkaefer) package/bin/0000755000175000017500000000000011737337551012175 5ustar daviddavidpackage/bin/node-get-file.js0000755000175000017500000000251311716556317015156 0ustar daviddavid#!/usr/bin/env node var path = require('path'), url = require('url'), util = require('util'), get = require('../lib/node-get/index.js'); var usage = 'usage:\n' + '\ndownload to a file:' + '\n\tnode-get-file.js ' + '\n\nget contents of file:' + '\n\tnode-get-file.js -' // Guessing destination filenames wget-style has never been // very robust, so require users to specify them. var obj = process.argv[2]; var dest = process.argv[3]; if (!(obj && dest)) { console.log(usage); process.exit(1); } // Initialize the download. try { var download = new get({ uri: obj }); } catch(e) { util.debug(e); process.exit(1); } if (dest == '-') { // Download to disk. download.asString(function(err, str) { // Print both errors and debugging messages // to stderr so that eventual piping is succesfull if (err) { util.debug(err); } else { console.log(str); } }); } else { // Download to disk. download.toDisk(dest, function(err, filename) { // Print both errors and debugging messages // to stderr so that eventual piping is succesfull if (err) { util.debug(err); } else { util.debug('Downloaded to ' + filename); } }); }