diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..79b5594 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +**/.DS_Store diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..5c6593a --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2014 TODO: Tyler Nappy + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index bcec6e2..9dd9cfe 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,199 @@ -# Node JS client library for IDOL OnDemand - -Basic library to help with calling IDOL OnDemand APIs [http://idolondemand.com](http://idolondemand.com) +**Note:** formerly known as `iod-node`. For the older version, see the `iod` branch. + +# Node JS client library for Haven OnDemand +Official client library to help with calling Haven OnDemand APIs [http://havenondemand.com](http://havenondemand.com). + +## What is Haven OnDemand? +Haven OnDemand is a set of over 60 APIs for handling all sorts of unstructured data. Here are just some of our APIs' capabilities: +* Speech to text +* OCR +* Text extraction +* Indexing documents +* Smart search +* Language identification +* Concept extraction +* Sentiment analysis +* Web crawlers +* Machine learning + +For a full list of all the APIs and to try them out, check out https://www.havenondemand.com/developer/apis ### Installation - +To install, run the following command: ``` -npm install git+ +npm install havenondemand +``` +If you want to install the latest module directly from Github, use the following command: +``` +npm install git+https://github.com/HP-Haven-OnDemand/havenondemand-node ``` -### Start - +### Include it ```js -var iod = require('iod-node') -client= new iod.IODClient('http://api.idolondemand.com','apikey') +var havenondemand = require('havenondemand') +var client = new havenondemand.HODClient(apikey, version, proxy) ``` +You can find your API key [here](https://www.haveondemand.com/account/api-keys.html) after signing up. -### Callbacks +`version` is an optional parameter (defaults to `'v1'`) and can be either `'v1'` or `'v2'`. +`proxy` is an optional parameter. Please set this if you're behind a firewall. Here is an example of iniating the client if you're using a proxy: +```js +var havenondemand = require('havenondemand') +var client = new havenondemand.HODClient('123456-asdf', 'v1', 'http://user:pass@proxy.server.com:3128') ``` +### Callback +We must define a callback function and pass it as an argument +```js var callback = function(err,resp,body){ console.log(body) } +var data = {'text' : 'I like cats'} +client.post('analyzesentiment', data, false, callback) ``` -We can define our callbacks as functions and pass them as arguments - +The order of the arguments is strict. It must be in the following order: +method("api_name", {params}, async=[true|false], callback_method) +```js +var data = {'text' : 'I like cats'} +client.post('analyzesentiment', data, false, callback) ``` -var data= {'text':'I like cats'} -client.call('analyzesentiment',callback,data) + +### GET request +APIs can be accessed via a GET request. +```js +var data = {'text' : 'I like cats'} +client.get('analyzesentiment', data, false, function(err, resp, body) { + if (!err) { + console.log(resp.body) + } +}) ``` -Or we can use the .on('data') hook to do the same thing. +### Async calls +While node will mostly deals with processes asynchronously, Haven OnDemand offers server side asynchronous call methods which should be used with large files and slow queries. Pass a boolean for the async parameter. The API response will return back a job ID which is used to check the status or result of your API request. +```js +var jobID +var data = {'text': 'I love dogs'} +client.post('analyzesentiment', data, true, function(err, resp, body) { + jobID = resp.body.jobID + console.log(jobID) +}) +``` +**(Recommended method)** To check the status of your API call, use the following code with the jobID from obtained from the async call above. This will tell you if it's still processing or if it's complete, and if so, it will return the result. +```js +client.getJobStatus(jobID, function(err, resp, body) { + console.log(resp.body) +}) ``` -client.call('analyzesentiment',data).on('data',callback) +Or, to check the result of your API call, use the following code with the jobID obtained from the async call. *Note: This method may timeout if your async API call is still processing.* +```js +client.getJobResult(jobID, function(err, resp, body) { + console.log(resp.body) +}) ``` -THe order of the arguments after the api name don't matter when passed. +### Posting files -``` -client.call('analyzesentiment',data,callback) +File posting is handled using the "file" parameter name which is used for all current file postings in Haven OnDemand +```js +var data = {'file' : 'test.txt'} +client.post('analyzesentiment', data, false, function(err, resp, body) { + if (err) { + console.log(err) + } else { + console.log(resp.body) + } +}) ``` +### Combinations -### Async calls - -While node will mostly deal with things asynchronously, IDOL OnDemand offers a servetr side asynchronous calls method which should be used with large files and slow queries. +Haven OnDemand allows to chain two ore more APIs together to create customizable, reusable services. These combinations enable one data input to have unlimited transformations and processing all from a single API call. +If you created a combination API name "sentimentanalysistoindex" which takes input as plain text. You can call the combination API from the code shown below: +```js +var params = {text : "Haven OnDemand is awesome."}; +client.post_combination('sentimentanalysistoindex', params, false, function(err, resp, body) { + if (err) { + console.log(err) + } else { + console.log(resp.body) + } +}) ``` -client.call('analyzesentiment',callback,data,true) +If you created a combination API name "sentimentanalysistoindex" which takes JSON input and presumed that the name of your input is "jsoninput". And your combination API was defined to parse the jsonContent similar to the format below. You can call the combination API from the code as follows: +```js +var jsonContent = '{"arrayinput":[{"content":"Haven OnDemand is awesome."},{"content":"Sentiment Analysis API is very usefule."}]}' +var params = {} +params.jsoninput = jsonContent +client.get_combination('sentimentanalysistoindex', params, false, function(err, resp, body) { + if (err) { + console.log(err) + } else { + console.log(resp.body) + } +}) +``` +If you created a combination API name "sentimentanalysistoindex" which takes a file input and presumed that the name of your input is "textfile". And your combination API was defined to take also the language configuration. You can call the combination API from the code as follows: +```js +var files = [{"textfile":"path/document.txt"}] +var params = {} +params.file = files +params.language = "eng" +client.post_combination('sentimentanalysistoindex', params, false, function(err, resp, body) { + if (err) { + console.log(err) + } else { + console.log(resp.body) + } +}) ``` -Pass a boolean for the async parameter. - +To find out more about combinations and how to create one, see [here](https://dev.havenondemand.com/combination/home). -### Posting files +### Batch jobs -File posting is handled using the "file" parameter name which is used for all current file postings in IDOL OnDemand +Haven OnDemand allows you to batch multiple API jobs in a single request using the Job API, for example, to analyze a batch of web pages, documents or social media messages where you need to analyze each text individually but want to be more efficient with your code, or where you want to execute multiple API calls on a single web page, document, or text. **Note: files are currently not supported in this wrapper for batch jobs.** +```js +var jobID +var data = [ + { "name": "analyzesentiment", + "version": "v1", + "params": { + 'text': 'I love dogs' + } + }, + { "name": "extractconcepts", + "version": "v1", + "params": { + "url": "http://en.wikipedia.org/wiki/United_Kingdom" + } + } + ] +client.batchJob(data, function(err, resp, body) { + jobID = resp.body.jobID + console.log(jobID) +}) + +// +// check result of async request with Status API after some time +// + +client.getJobStatus(jobID, function(err, resp, body) { + console.log(resp.body) +}) ``` -var data={'file':'test.txt'} -client.call('analyzesentiment',data,callback) -``` + + +## Contributing +We encourage you to contribute to this repo! Please send pull requests with modified and updated code. + +1. Fork it ( https://github.com/HPE-Haven-OnDemand/havenondemand-node/fork ) +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create a new Pull Request diff --git a/index.js b/index.js index a407198..c392e19 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,3 @@ -var iod = require('./lib/iodneedle') +var hod = require('./lib/hodneedle') -module.exports=iod +module.exports=hod diff --git a/lib/hodneedle.js b/lib/hodneedle.js new file mode 100644 index 0000000..73360b3 --- /dev/null +++ b/lib/hodneedle.js @@ -0,0 +1,445 @@ +var needle = require('./needle/lib/needle') +var util = require('util') +var fs = require('fs') +var querystring = require('querystring') + +// Constructor +function HODClient(apikey, version, proxy, staging) { + // always initialize all instance properties + if (apikey=='http://api.havenondemand.com' || apikey=='http://api.havenondemand.com/' || apikey=='https://api.havenondemand.com' || apikey=='https://api.havenondemand.com/') { + throw new Error("Using an outdated wrapper constructor method. No need to include API URL.\nInclude as such:\n client = new havenondemand.HODClient(API_KEY, VERSION)") + } + if (version==undefined) {this.version="v1";} + else {this.version=version;} + this.apikey = apikey; + if (staging==undefined || staging==false) { + this.endpoint = "https://api.havenondemand.com/1/api/%s/%s/"+this.version; + } else if (staging==true) { + this.endpoint = "https://api.staging.havenondemand.com/1/api/%s/%s/"+this.version; + } + this.proxy = proxy; + this.getJobResultEndpoint = "https://api.havenondemand.com/1/job/result/%s?apikey=%s"; + this.getJobStatusEndpoint = "https://api.havenondemand.com/1/job/status/%s?apikey=%s"; + this.jobAPIEndpoint = "https://api.havenondemand.com/1/job"; +} +// class methods + +needle.defaults({ timeout: 120000}); + +validadeParameters=function(hodApp,params,async,callback) { + if (typeof hodApp == "undefined") { + throw new Error("Missing hodApp parameter. Required valid API name") + } + if (typeof params == "undefined") { + throw new Error("Missing params parameter. Required API's input and configuration parameters") + } + if (typeof async == "undefined") { + throw new Error("Missing async parameter. Required true or false") + }else if (typeof async == "function"){ + if (typeof callback == "undefined") { + throw new Error("Missing async parameter. Required true or false") + }else{ + throw new Error("Wrong parameters order. Required (hodApp,params,async,callback)") + } + } + if (typeof callback == "undefined") { + throw new Error("Missing callback parameter. Required callback function") + }else if (typeof callback != "function"){ + throw new Error("Wrong callback parameter. Required callback function") + } +} +isJSON=function(value) { + var ret = true + try { + JSON.parse(value); + } catch (e) { + ret = false + } + return ret +} +HODClient.prototype.post_combination = function(hodApp,params,async,callback) { + var data = {} + validadeParameters(hodApp,params,async,callback) + async_string = "sync"; + data.apikey = this.apikey; + data.combination = hodApp; + for(var item in params){ + data[item] = params[item] + } + + if (async) async_string="async"; + var url = util.format(this.endpoint, async_string, "executecombination"); + + if (async){ + var callback = callback; + var callbackmanager=function(err,resp,body){ + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.'; + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + body={'async':true,'data':body}; + body.status + callback(error, resp, body); + } + } + else{ + var callbackmanager = function(err, resp, body) { + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.' + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + callback(error, resp, body); + } + } + if (this.proxy != undefined) { + needle.post(url, data, { combination:true, multipart: true, proxy: this.proxy }, callbackmanager); + } else { + needle.post(url, data, { combination:true, multipart: true }, callbackmanager); + } +} + +HODClient.prototype.get_combination = function(hodApp,params,async,callback) { + validadeParameters(hodApp,params,async,callback) + var data= {}; + async_string="sync"; + data.apikey=this.apikey; + + if (async) { + async_string="async"; + } + var configs = util.format("apikey=%s&combination=%s", this.apikey, hodApp); + + for(var item in params){ + if (item == 'file'){ + throw new Error("Cannot perform GET with file. Use POST request.") + } + else { + if (isJSON(params[item])) + configs += util.format('¶meters={"name":"%s","value":%s}', item, params[item]) + else + configs += util.format('¶meters={"name":"%s","value":"%s"}', item, params[item]) + } + } + var url = util.format(this.endpoint, async_string,"executecombination") + "?" + configs + + if (async){ + var callback = callback; + var callbackmanager=function(err,resp,body){ + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.'; + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + body={'async':true,'data':body}; + body.status + callback(error, resp, body); + } + } + else{ + var callbackmanager = function(err, resp, body) { + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.'; + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + callback(error, resp, body); + } + } + if (this.proxy != undefined) { + needle.get(url, { proxy: this.proxy }, callbackmanager); + } else { + needle.get(url, callbackmanager); + } +}; +HODClient.prototype.post = function(hodApp,params,async,callback) { + validadeParameters(hodApp,params,async,callback) + var data = params + if (data["file"]){ + data.file={'file':data["file"],'content_type':'multipart/form-data'} + } + async_string="sync"; + data.apikey=this.apikey; + + if (async) { + async_string="async"; + } + + var url = util.format(this.endpoint,async_string,hodApp); + + if (async){ + var callback = callback; + var callbackmanager=function(err,resp,body){ + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.'; + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + body={'async':true,'data':body}; + body.status + callback(error, resp, body); + } + } + else{ + var callbackmanager = function(err, resp, body) { + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.' + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + callback(error, resp, body); + } + } + if (this.proxy != undefined) { + needle.post(url, data, { multipart: true, proxy: this.proxy }, callbackmanager); + } else { + needle.post(url, data, { multipart: true }, callbackmanager); + } +}; + +HODClient.prototype.get = function(hodApp,params,async,callback) { + validadeParameters(hodApp,params,async,callback) + if (params["file"]){ + throw new Error("Cannot perform GET with file. Use POST request.") + } + async_string="sync"; + params.apikey=this.apikey; + + if (async) { + async_string="async"; + } + + var url = util.format(this.endpoint, async_string,hodApp) + "?" + querystring.stringify(params) + if (async){ + var callback = callback; + var callbackmanager=function(err,resp,body){ + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.'; + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + body={'async':true,'data':body}; + body.status + callback(error, resp, body); + } + } + else{ + var callbackmanager = function(err, resp, body) { + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.'; + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + callback(error, resp, body); + } + } + if (this.proxy != undefined) { + needle.get(url, { proxy: this.proxy }, callbackmanager); + } else { + needle.get(url, callbackmanager); + } +}; + +validadeGetJob=function(jobID,callback) { + if (typeof jobID == "undefined") { + if (typeof callback == "undefined") { + throw new Error("Missing jobID parameter. Required valid job ID") + }else{ + throw new Error("Wrong parameters order. Required function(jobID,callback)") + } + }else if (typeof jobID == "function"){ + if (typeof callback == "undefined") { + throw new Error("Missing jobID parameter. Required valid job ID") + }else{ + throw new Error("Wrong parameters order. Required function(jobID,callback)") + } + } + if (typeof callback == "undefined") { + throw new Error("Missing callback parameter. Required callback function") + }else if (typeof callback != "function"){ + throw new Error("Wrong callback parameter. Required callback function") + } +} +HODClient.prototype.getJobResult = function(jobID,callback) { + validadeGetJob(jobID,callback) + var url = util.format(this.getJobResultEndpoint, jobID, this.apikey); + + if (this.proxy != undefined) { + return needle.get(url, {proxy: this.proxy}, callback); + } else { + return needle.get(url, callback); + } +} + +HODClient.prototype.getJobStatus = function(jobID,callback) { + validadeGetJob(jobID,callback) + var url = util.format(this.getJobStatusEndpoint, jobID, this.apikey); + + if (this.proxy != undefined) { + return needle.get(url, {proxy: this.proxy}, callback); + } else { + return needle.get(url, callback); + } +} + +validadeBatchJob=function(params, callback) { + if (typeof params == "undefined") { + if (typeof callback == "undefined") { + throw new Error("Missing params parameter. Required valid batch job parameters") + }else{ + throw new Error("Wrong parameters order. Required function(params,callback)") + } + }else if (typeof params == "function"){ + if (typeof callback == "undefined") { + throw new Error("Missing params parameter. Required valid batch job parameters") + }else{ + throw new Error("Wrong parameters order. Required function(params,callback)") + } + } + if (typeof callback == "undefined") { + throw new Error("Missing callback parameter. Required callback function") + }else if (typeof callback != "function"){ + throw new Error("Wrong callback parameter. Required callback function") + } +} +HODClient.prototype.batchJob=function(params, callback) { // data is array + validadeBatchJob(params,callback) + var data = {}; + var url = this.jobAPIEndpoint; + var apikey = this.apikey + + if (typeof params =="undefined") { + data={}; + } else { + processData(params, function(processedData) { //processedData is array + data.apikey= apikey; + data["job"] = processedData; + var callbackmanager=function(err,resp,body){ + var error; + if (typeof(resp) == 'undefined') { + error = 'Problem getting result from Haven OnDemand. Please try aagain.' + } else if (resp.body.error) { + error = resp.body; + } else { + error = null; + } + body={'async':true,'data':body}; + body.status + callback(error, resp, body); + } + + if (this.proxy != undefined) { + needle.post(url, data, { multipart: true, proxy: this.proxy }, callbackmanager); + } else { + needle.post(url, data, { multipart: true }, callbackmanager); + } + }) + } +} + +processData = function(params, callback) { + for (var i=0; i Api changes between v0.6 and v0.7 +--------------------------------------------------- + +In version 0.7 a lot of work was done on the Needle internals to make streams a first class citizen. Needle can now be used as a streams2-compatible stream and you gain a lot of performance improvements come with it. + +While great care was taken not to introduce any breaking changes, there are probably a few edge cases in which Needle's behaviour has changed, specifically: + + * Needle now emits a strict streams2-compatible stream. This means that if your code relied on the Needle stream to always be in flowing mode, your code will likely need an update. For more information about this new Stream behavior, please refer to [the Node.JS blog](http://blog.nodejs.org/2012/12/20/streams2/). + + * In the v0.6 release, Needle's stream didn't parse, uncompress or did anything to the body content: everything chunk of data that was emitted on the stream was the raw body. In the 0.7 release every chunk of data will be a fully processed chunk, including uncompression, character recoding and parsing (in case of XML/JSON). + +If you use the regular callback interface of Needle, this will be a backwards-compatible upgrade. diff --git a/lib/needle/README.md b/lib/needle/README.md new file mode 100644 index 0000000..84e1151 --- /dev/null +++ b/lib/needle/README.md @@ -0,0 +1,439 @@ +Needle +====== +This is a modified version for HPE Haven OnDemand. + +[![NPM](https://nodei.co/npm/needle.png)](https://nodei.co/npm/needle/) + +The leanest and most handsome HTTP client in the Nodelands. + +```js +var needle = require('needle'); + +needle.get('http://www.google.com', function(error, response) { + if (!error && response.statusCode == 200) + console.log(response.body); +}); +``` + +Callbacks not floating your boat? Needle got your back. + +```js +var data = { + file: '/home/johnlennon/walrus.png', + content_type: 'image/png' +}; + +needle + .post('https://my.server.com/foo', data, { multipart: true }) + .on('readable', function() { /* eat your chunks */ }) + .on('end', function() { + console.log('Ready-o, friend-o.'); + }) +``` + +With only one single dependency, Needle supports: + + - HTTP/HTTPS requests, with the usual verbs you would expect. + - All of Node's native TLS options, such as 'rejectUnauthorized' (see below). + - Basic & Digest authentication + - Multipart form-data (e.g. file uploads) + - HTTP Proxy forwarding, optionally with authentication. + - Streaming gzip or deflate decompression + - Automatic XML & JSON parsing + - 301/302 redirect following, if enabled, and + - Streaming non-UTF-8 charset decoding, via `iconv-lite`. + +And yes, Mr. Wayne, it does come with the latest streams2 support. + +This makes Needle an ideal alternative for performing quick HTTP requests in Node, either for API interaction, downloading or uploading streams of data, and so on. If you need OAuth, AWS support or anything fancier, you should check out mikeal's request module. + +Important +--------- + +The version bump from 0.6 to 0.7 includes a few notable changes to the streaming interface. If you were using Needle in 'steams mode', please take a look at the [changelog](https://github.com/tomas/needle/blob/master/CHANGELOG.md) to see what's going on. If you were using regular callbacks, no problemo amigo -- you can update to 0.7+ and everything will be smooth as silk. + +Install +------- + +``` +$ npm install needle +``` + +Usage +----- + +```js +// using callback +needle.get('ifconfig.me/all.json', function(error, response) { + if (!error) + console.log(response.body.ip_addr); // JSON decoding magic. :) +}); + +// using streams +var out = fs.createWriteStream('logo.png'); +needle.get('https://google.com/images/logo.png').pipe(out); +``` + +As you can see, you can call Needle with a callback or without it. When passed, the response body will be buffered and written to `response.body`, and the callback will be fired when all of the data has been collected and processed (e.g. decompressed, decoded and/or parsed). + +When no callback is passed, the buffering logic will be skipped but the response stream will still go through Needle's processing pipeline, so you get all the benefits of post-processing while keeping the streamishness we all love from Node. + +Response pipeline +----------------- + +Depending on the response's Content-Type, Needle will either attempt to parse JSON or XML streams, or, if a text response was received, will ensure that the final encoding you get is UTF-8. For XML decoding to work, though, you'll need to install the `xml2js` package as we don't enforce unneeded dependencies unless strictly needed. + +You can also request a gzip/deflated response, which, if sent by the server, will be processed before parsing or decoding is performed. + +```js +needle.get('http://stackoverflow.com/feeds', { compressed: true }, function(err, resp) { + console.log(resp.body); // this little guy won't be a Gzipped binary blob + // but a nice object containing all the latest entries +}); +``` + +Or in anti-callback mode, using a few other options: + +```js +var options = { + compressed : true, + follow : true, + rejectUnauthorized : true +} + +// in this case, we'll ask Needle to follow redirects (disabled by default), +// but also to verify their SSL certificates when connecting. +var stream = needle.get('https://backend.server.com/everything.html', options); + +stream.on('readable', function() { + while (data = this.read()) { + console.log(data.toString()); + } +}) +``` + +API +--- + +All of Needle's request methods return a Readable stream, and both `options` and `callback` are optional. If passed, the callback will return three arguments: `error`, `response` and `body`, which is basically an alias for `response.body`. + +### needle.head(url, options, callback) + +```js +var options = { + timeout: 5000 // if we don't get a response in 5 seconds, boom. +} + +needle.head('https://my.backend.server.com', function(err, resp) { + if (err) + console.log('Shoot! Something is wrong: ' + err.message) + else + console.log('Yup, still alive.') +}) +``` + +### needle.get(url, options, callback) + +```js +needle.get('google.com/search?q=syd+barrett', function(err, resp) { + // if no http:// is found, Needle will automagically prepend it. +}); +``` + +### needle.post(url, data, options, callback) + +```js +var options = { + headers: { 'X-Custom-Header': 'Bumbaway atuna' } +} + +needle.post('https://my.app.com/endpoint', 'foo=bar', options, function(err, resp) { + // you can pass params as a string or as an object. +}); +``` + +### needle.put(url, data, options, callback) + +```js +var nested = { + params: { + are: { + also: 'supported' + } + } +} + +needle.put('https://api.app.com/v2', nested, function(err, resp) { + console.log('Got ' + resp.bytes + ' bytes.') // another nice treat from this handsome fella. +}); +``` + +### needle.delete(url, data, options, callback) + +```js +var options = { + username: 'fidelio', + password: 'x' +} + +needle.delete('https://api.app.com/messages/123', null, options, function(err, resp) { + // in this case, data may be null, but you need to explicity pass it. +}); +``` + +### needle.request(method, url, data, options, callback) + +Generic request. This not only allows for flexibility, but also lets you perform a GET request with data, in which case will be appended to the request as a query string. + +```js +var data = { + q : 'a very smart query', + page : 2, + format : 'json' +} + +needle.request('get', 'forum.com/search', data, function(err, resp) { + if (!err && resp.statusCode == 200) + console.log(resp.body); // here you go, mister. +}); +``` + +More examples after this short break. + +Request options +--------------- + + - `timeout` : Returns error if no response received in X milisecs. Defaults to `10000` (10 secs). `0` means no timeout. + - `follow` : Number of redirects to follow. `false` means don't follow any (default), `true` means 10. + - `multipart` : Enables multipart/form-data encoding. Defaults to `false`. Use it when uploading files. + - `proxy` : Forwards request through HTTP(s) proxy. Eg. `proxy: 'http://proxy.server.com:3128'` + - `agent` : Uses an http.Agent of your choice, instead of the global, default one. + - `headers` : Object containing custom HTTP headers for request. Overrides defaults described below. + - `auth` : Determines what to do with provided username/password. Options are `auto`, `digest` or `basic` (default). `auto` will detect the type of authentication depending on the response headers. + - `json` : When `true`, sets content type to `application/json` and sends request body as JSON string, instead of a query string. + +Response options +---------------- + + - `decode` : Whether to decode the text responses to UTF-8, if Content-Type header shows a different charset. Defaults to `true`. + - `parse` : Whether to parse XML or JSON response bodies automagically. Defaults to `true`. + - `output` : Dump response output to file. This occurs after parsing and charset decoding is done. + +Note: To stay light on dependencies, Needle doesn't include the `xml2js` module used for XML parsing. To enable it, simply do `npm install xml2js`. + +HTTP Header options +------------------- + +These are basically shortcuts to the `headers` option described above. + + - `compressed`: If `true`, sets 'Accept-Encoding' header to 'gzip,deflate', and inflates content if zipped. Defaults to `false`. + - `username` : For HTTP basic auth. + - `password` : For HTTP basic auth. Requires username to be passed, but is optional. + - `accept` : Sets 'Accept' HTTP header. Defaults to `*/*`. + - `connection`: Sets 'Connection' HTTP header. Defaults to `close`. + - `user_agent`: Sets the 'User-Agent' HTTP header. Defaults to `Needle/{version} (Node.js {node_version})`. + +Node.js TLS Options +------------------- + +These options are passed directly to `https.request` if present. Taken from the [original documentation](http://nodejs.org/docs/latest/api/https.html): + + - `pfx`: Certificate, Private key and CA certificates to use for SSL. + - `key`: Private key to use for SSL. + - `passphrase`: A string of passphrase for the private key or pfx. + - `cert`: Public x509 certificate to use. + - `ca`: An authority certificate or array of authority certificates to check the remote host against. + - `ciphers`: A string describing the ciphers to use or exclude. + - `rejectUnauthorized`: If true, the server certificate is verified against the list of supplied CAs. An 'error' event is emitted if verification fails. Verification happens at the connection level, before the HTTP request is sent. + - `secureProtocol`: The SSL method to use, e.g. SSLv3_method to force SSL version 3. + +Overriding Defaults +------------------- + +Yes sir, we have it. Needle includes a `defaults()` method, that lets you override some of the defaults for all future requests. Like this: + +```js +needle.defaults({ timeout: 60000, user_agent: 'MyApp/1.2.3' }); +``` + +This will override Needle's default user agent and 10-second timeout, so you don't need to pass those options in every other request. + +Examples Galore +--------------- + +### HTTPS GET with Basic Auth + +```js +needle.get('https://api.server.com', { username: 'you', password: 'secret' }, + function(err, resp) { + // used HTTP auth +}); +``` + +Or use [RFC-1738](http://tools.ietf.org/html/rfc1738#section-3.1) basic auth URL syntax: + +```js +needle.get('https://username:password@api.server.com', function(err, resp) { + // used HTTP auth from URL +}); +``` + +### Digest Auth + +```js +needle.get('other.server.com', { username: 'you', password: 'secret', auth: 'digest' }, + function(err, resp, body) { + // needle prepends 'http://' to your URL, if missing +}); +``` + +### Custom Accept header, deflate + +```js +var options = { + compressed : true, + follow : true, + accept : 'application/vnd.github.full+json' +} + +needle.get('api.github.com/users/tomas', options, function(err, resp, body) { + // body will contain a JSON.parse(d) object + // if parsing fails, you'll simply get the original body +}); +``` + +### GET XML object + +```js +needle.get('https://news.ycombinator.com/rss', function(err, resp, body) { + // if xml2js is installed, you'll get a nice object containing the nodes in the RSS +}); +``` + +### GET binary, output to file + +```js +needle.get('http://upload.server.com/tux.png', { output: '/tmp/tux.png' }, function(err, resp, body) { + // you can dump any response to a file, not only binaries. +}); +``` + +### GET through proxy + +```js +needle.get('http://search.npmjs.org', { proxy: 'http://localhost:1234' }, function(err, resp, body) { + // request passed through proxy +}); +``` + +### GET a very large document in a stream (from 0.7+) + +```js +var stream = needle.get('http://www.as35662.net/100.log'); + +stream.on('readable', function() { + var chunk; + while (chunk = this.read()) { + console.log('got data: ', chunk); + } +}); +``` + +### GET JSON object in a stream (from 0.7+) + +```js +var stream = needle.get('http://jsonplaceholder.typicode.com/db', { parse: true }); + +stream.on('readable', function() { + var node; + + // our stream will only emit a single JSON root node. + while (node = this.read()) { + console.log('got data: ', node); + } +}); +``` + +### GET JSONStream flexible parser with search query (from 0.7+) + +```js + + // The 'data' element of this stream will be the string representation + // of the titles of all posts. + +needle.get('http://jsonplaceholder.typicode.com/db', { parse: true }) + .pipe(new JSONStream.parse('posts.*.title')); + .on('data', function (obj) { + console.log('got post title: %s', obj); + }); +``` + +### File upload using multipart, passing file path + +```js +var data = { + foo: 'bar', + image: { file: '/home/tomas/linux.png', content_type: 'image/png' } +} + +needle.post('http://my.other.app.com', data, { multipart: true }, function(err, resp, body) { + // needle will read the file and include it in the form-data as binary +}); +``` + +### Stream upload, PUT or POST + +``` js +needle.put('https://api.app.com/v2', fs.createReadStream('myfile.txt'), function(err, resp, body) { + // stream content is uploaded verbatim +}); +``` + +### Multipart POST, passing data buffer + +```js +var buffer = fs.readFileSync('/path/to/package.zip'); + +var data = { + zip_file: { + buffer : buffer, + filename : 'mypackage.zip', + content_type : 'application/octet-stream' + } +} + +needle.post('http://somewhere.com/over/the/rainbow', data, { multipart: true }, function(err, resp, body) { + // if you see, when using buffers we need to pass the filename for the multipart body. + // you can also pass a filename when using the file path method, in case you want to override + // the default filename to be received on the other end. +}); +``` + +### Multipart with custom Content-Type + +```js +var data = { + token: 'verysecret', + payload: { + value: JSON.stringify({ title: 'test', version: 1 }), + content_type: 'application/json' + } +} + +needle.post('http://test.com/', data, { timeout: 5000, multipart: true }, function(err, resp, body) { + // in this case, if the request takes more than 5 seconds + // the callback will return a [Socket closed] error +}); +``` + +For even more examples, check out the examples directory in the repo. + +Credits +------- + +Written by Tomás Pollak, with the help of contributors. + +Copyright +--------- + +(c) 2014 Fork Ltd. Licensed under the MIT license. diff --git a/lib/needle/lib/auth.js b/lib/needle/lib/auth.js new file mode 100644 index 0000000..0f4b7bb --- /dev/null +++ b/lib/needle/lib/auth.js @@ -0,0 +1,87 @@ +var createHash = require('crypto').createHash; + +var md5 = function(string) { + return createHash('md5').update(string).digest('hex'); +} + +var basic = function(user, pass) { + var str = typeof pass == 'undefined' ? user : [user, pass].join(':'); + return 'Basic ' + new Buffer(str).toString('base64'); +} + +// digest logic inspired from https://github.com/simme/node-http-digest-client +var digest = {}; + +digest.parse_header = function(header) { + var challenge = {}, + matches = header.match(/([a-z0-9_-]+)="([^"]+)"/gi); + + for (var i = 0, l = matches.length; i < l; i++) { + var pos = matches[i].indexOf('='), + key = matches[i].substring(0, pos), + val = matches[i].substring(pos + 1); + challenge[key] = val.substring(1, val.length - 1); + } + + return challenge; +} + +digest.update_nc = function(nc) { + var max = 99999999; + nc++; + + if (nc > max) + nc = 1; + + var padding = new Array(8).join('0') + ''; + nc = nc + ''; + return padding.substr(0, 8 - nc.length) + nc; +} + +digest.generate = function(header, user, pass, method, path) { + + var nc = 1, + cnonce = null, + challenge = digest.parse_header(header); + + var ha1 = md5(user + ':' + challenge.realm + ':' + pass), + ha2 = md5(method + ':' + path), + resp = [ha1, challenge.nonce]; + + if (typeof challenge.qop === 'string') { + cnonce = md5(Math.random().toString(36)).substr(0, 8); + nc = digest.update_nc(nc); + resp = resp.concat(nc, cnonce); + } + + resp = resp.concat(challenge.qop, ha2); + + var params = { + username: user, + realm: challenge.realm, + nonce: challenge.nonce, + uri: path, + qop: challenge.qop, + response: md5(resp.join(':')) + } + +// if (challenge.opaque) { +// params.opaque = challenge.opaque; +// } + + if (cnonce) { + params.nc = nc; + params.cnonce = cnonce; + } + + header = [] + for (var k in params) + header.push(k + '="' + params[k] + '"') + + return 'Digest ' + header.join(', '); +} + +module.exports = { + basic: basic, + digest: digest.generate +} diff --git a/lib/needle/lib/decoder.js b/lib/needle/lib/decoder.js new file mode 100644 index 0000000..c63a56d --- /dev/null +++ b/lib/needle/lib/decoder.js @@ -0,0 +1,52 @@ +var iconv, + inherits = require('util').inherits, + stream = require('stream'); + +var charsetReg = /(?:charset|encoding)\s*=\s*['"]? *([\w\-]+)/i; + +inherits(StreamDecoder, stream.Transform); + +function StreamDecoder(charset) { + if (!(this instanceof StreamDecoder)) + return new StreamDecoder(charset); + + stream.Transform.call(this, charset); + this.charset = charset; + this.parsedFromChunk = false; +} + +StreamDecoder.prototype._transform = function(chunk, encoding, done) { + var res; + + // try get charset from chunk, just once + if (this.charset == 'iso-8859-1' && !this.parsedFromChunk) { + this.parsedFromChunk = true; + var matchs = charsetReg.exec(chunk.toString()); + if (matchs) { + var cs = matchs[1].toLowerCase(); + this.charset = cs == 'utf-8' ? 'utf8' : cs; + } + } + + try { + res = iconv.decode(chunk, this.charset); + } catch(e) { // something went wrong, just return original chunk + res = chunk; + } + + this.push(res); + done(); +} + +module.exports = function(charset) { + try { + if (!iconv) iconv = require('iconv-lite'); + } catch(e) { + /* iconv not found */ + } + + if (iconv) + return new StreamDecoder(charset); + else + return new stream.PassThrough; +} diff --git a/lib/needle/lib/multipart.js b/lib/needle/lib/multipart.js new file mode 100644 index 0000000..32fa1b6 --- /dev/null +++ b/lib/needle/lib/multipart.js @@ -0,0 +1,163 @@ +var readFile = require('fs').readFile, + basename = require('path').basename; +var util = require('util') + +isJSON=function(value) { + var ret = true + try { + JSON.parse(value); + } catch (e) { + ret = false + } + return ret +} +exports.build_combination = function(data, boundary, callback) { + if (typeof data != 'object') + return callback(new Error('Multipart builder expects data as key/val object.')); + + var body = '', + count = Object.keys(data).length; + if (count === 0) + return callback(new Error('Empty multipart body. Invalid data.')) + + var done = function(err, section) { + if (err) return callback(err); + if (section) body += section; + --count || callback(null, body + '--' + boundary + '--'); + }; + + var extra = function(err, section) { + if (err) return callback(err); + if (section) + body += section; + }; + + for (var key in data) { + var value = data[key]; + if (value === null || typeof value == 'undefined') { + done(); + }else { + if (key == 'apikey' || key == 'combination'){ + var part = (value.buffer || value.file || value.content_type) ? value : {value: value}; + generate_part(key, part, boundary, done); + }else if (key == 'file'){ + var len = value.length + for(var i=0; i +// (c) 2012-2013 - Fork Ltd. +// MIT Licensed +////////////////////////////////////////// + +var fs = require('fs'), + http = require('http'), + https = require('https'), + url = require('url'), + stream = require('stream'), + stringify = require('./querystring').build, + multipart = require('./multipart'), + auth = require('./auth'), + parsers = require('./parsers'), + decoder = require('./decoder'); + +////////////////////////////////////////// +// variabilia +////////////////////////////////////////// + +var version = JSON.parse(fs.readFileSync(__dirname + '/../package.json').toString()).version, + debugging = !!process.env.DEBUG, + debug = debugging ? console.log : function() { /* noop */ }; + +var user_agent = 'Needle/' + version; +user_agent += ' (Node.js ' + process.version + '; ' + process.platform + ' ' + process.arch + ')'; + +var node_tls_opts = 'agent pfx key passphrase cert ca ciphers rejectUnauthorized secureProtocol'; + +////////////////////////////////////////// +// decompressors for gzip/deflate bodies +////////////////////////////////////////// + +var decompressors = {}; + +try { + + var zlib = require('zlib') + + decompressors['x-deflate'] = zlib.Inflate; + decompressors['deflate'] = zlib.Inflate; + decompressors['x-gzip'] = zlib.Gunzip; + decompressors['gzip'] = zlib.Gunzip; + +} catch(e) { /* zlib not available */ } + +////////////////////////////////////////// +// defaults +////////////////////////////////////////// + +var defaults = { + accept : '*/*', + connection : 'close', + user_agent : user_agent, + follow : 0, + decode_response : true, + parse_response : true, + compressed : false, + timeout : 10000, + encoding : 'utf8', + boundary : '--------------------NODENEEDLEHTTPCLIENT' +} + +////////////////////////////////////////// +// the main act +////////////////////////////////////////// + +var Needle = { + request: function(method, uri, data, options, callback) { + var self = this, + out = new stream.PassThrough({ objectMode: false }), + callback = (typeof options == 'function') ? options : callback, + options = options || {}; + + // uri checks and parsing + if (typeof uri !== 'string') + throw new TypeError('URL must be a string, not ' + uri); + + // if no 'http' is found on URL, prepend it. + if (uri.indexOf('http') === -1) + uri = 'http://' + uri; + + // if url contains user:pass@host, parse it. + if (uri.indexOf('@') !== -1) { + var parts = (url.parse(uri).auth || '').split(':'); + options.username = parts[0]; + options.password = parts[1]; + } + + var config = { + base_opts : {}, + proxy : options.proxy, + output : options.output, + encoding : options.encoding || (options.multipart ? 'binary' : defaults.encoding), + decode_response : options.decode === false ? false : defaults.decode_response, + parse_response : options.parse === false ? false : defaults.parse_response, + follow : options.follow === true ? 10 : typeof options.follow == 'number' ? options.follow : defaults.follow, + timeout : (typeof options.timeout == 'number') ? options.timeout : defaults.timeout + } + + // if any of node's TLS options are passed, let them be passed to https.request() + node_tls_opts.split(' ').forEach(function(key) { + if (typeof options[key] != 'undefined') { + config.base_opts[key] = options[key]; + if (typeof options.agent == 'undefined') + config.base_opts.agent = false; // otherwise tls options are skipped + } + }); + + config.headers = { + 'Accept' : options.accept || defaults.accept, + 'Connection' : options.connection || defaults.connection, + 'User-Agent' : options.user_agent || defaults.user_agent + } + + if ((options.compressed || defaults.compressed) && typeof zlib != 'undefined') + config.headers['Accept-Encoding'] = 'gzip,deflate'; + + for (var h in options.headers) + config.headers[h] = options.headers[h]; + + if (options.username) { + if (options.auth && (options.auth == 'auto' || options.auth == 'digest')) { + config.credentials = [options.username, options.password]; + } else { + var auth_header = options.proxy ? 'Proxy-Authorization' : 'Authorization'; + config.headers[auth_header] = auth.basic(options.username, options.password); + } + } + + if (data) { + if (method.toUpperCase() == 'GET') { // build query string and append to URI + uri = uri.replace(/\?.*|$/, '?' + stringify(data)); + post_data = null; + + } else if (options.multipart) { // build multipart body for request + var boundary = options.boundary || defaults.boundary; + + if (options.combination){ + multipart.build_combination(data, boundary, function(err, body) { + if (err) throw(err); + + config.headers['Content-Type'] = 'multipart/form-data; boundary=' + boundary; + config.headers['Content-Length'] = body.length; + self.send_request(1, method, uri, config, body, out, callback); + }); + }else { + multipart.build(data, boundary, function(err, body) { + if (err) throw(err); + + config.headers['Content-Type'] = 'multipart/form-data; boundary=' + boundary; + config.headers['Content-Length'] = body.length; + self.send_request(1, method, uri, config, body, out, callback); + }); + } + return out; // stream + + } else if (this.is_stream(data) || Buffer.isBuffer(data)) { + + post_data = data; + + } else { // string or object data, no multipart. + + // format data according to content type + var post_data = (typeof(data) === 'string') ? data : + options.json ? JSON.stringify(data) : stringify(data); + + // if no content-type was passed, determine if json or not. + if (!config.headers['Content-Type']) { + config.headers['Content-Type'] = options.json + ? 'application/json' + : 'application/x-www-form-urlencoded'; + } + + post_data = new Buffer(post_data, config.encoding); + config.headers['Content-Length'] = post_data.length; + + // unless a specific accept header was passed, assume json wants json back. + if (options.json && config.headers['Accept'] === defaults.accept) + config.headers['Accept'] = 'application/json'; + } + } + return this.send_request(1, method, uri, config, post_data, out, callback); + }, + + get_request_opts: function(method, uri, config) { + var opts = config.base_opts, + proxy = config.proxy, + remote = proxy ? url.parse(proxy) : url.parse(uri); + + opts.protocol = remote.protocol; + opts.host = remote.hostname; + opts.port = remote.port || (remote.protocol == 'https:' ? 443 : 80); + opts.path = proxy ? uri : remote.pathname + (remote.search || ''); + opts.method = method; + opts.headers = config.headers; + + if (!opts.headers['Host']) { + // if using proxy, make sure the host header shows the final destination + var target = proxy ? url.parse(uri) : remote; + opts.headers['Host'] = target.hostname; + + // and if a non standard port was passed, append it to the port header + if (target.port && [80, 443].indexOf(target.port) === -1) { + opts.headers['Host'] += ':' + target.port; + } + } + + return opts; + }, + + get_auth_header: function(header, credentials, request_opts) { + var type = header.split(' ')[0], + user = credentials[0], + pass = credentials[1]; + + if (type == 'Digest') { + return auth.digest(header, user, pass, request_opts.method, request_opts.path); + } else if (type == 'Basic') { + return auth.basic(user, pass); + } + }, + + send_request: function(count, method, uri, config, post_data, out, callback) { + var timer, + returned = 0, + self = this, + request_opts = this.get_request_opts(method, uri, config), + protocol = request_opts.protocol == 'https:' ? https : http; + + var done = function(err, resp, body) { + if (returned++ > 0) return; + if (callback) + callback(err, resp, body); + else + out.emit('end', err, resp, body); + } + + debug('Making request #' + count, request_opts); + var request = protocol.request(request_opts, function(resp) { + + var headers = resp.headers; + debug('Got response', headers); + if (timer) clearTimeout(timer); + + // if redirect code is found, send a GET request to that location if enabled via 'follow' option + if ([301, 302, 303].indexOf(resp.statusCode) != -1 && headers.location) { + if (count <= config.follow) { + out.emit('redirect', headers.location); + delete config.headers['Content-Length']; // in case the original was a multipart POST request. + return self.send_request(++count, 'GET', url.resolve(uri, headers.location), config, null, out, callback); + } else if (config.follow > 0) { + return done(new Error('Max redirects reached. Possible loop in: ' + headers.location)); + } + } + + // if authentication is requested and credentials were not passed, resend request if we have user/pass + if (resp.statusCode == 401 && headers['www-authenticate'] && config.credentials) { + if (!config.headers['Authorization']) { // only if authentication hasn't been sent + var auth_header = self.get_auth_header(headers['www-authenticate'], config.credentials, request_opts); + + if (auth_header) { + config.headers['Authorization'] = auth_header; + return self.send_request(count, method, uri, config, post_data, out, callback); + } + } + } + + // ok so we got a valid (non-redirect & authorized) response. notify the stream guys. + out.emit('headers', headers); + + var pipeline = [], + parsed = false, + mime = self.parse_content_type(headers['content-type']), + text_response = mime.type && mime.type.indexOf('text/') != -1; + + // To start, if our body is compressed and we're able to inflate it, do it. + if (headers['content-encoding'] && decompressors[headers['content-encoding']]) { + pipeline.push(decompressors[headers['content-encoding']]()); + } + + // If parse is enabled and we have a parser for it, then go for it. + if (config.parse_response && parsers[mime.type]) { + parsed = true; + pipeline.push(parsers[mime.type]()); + + // set objectMode on out stream to improve performance + out._writableState.objectMode = true; + out._readableState.objectMode = true; + + // If we're not parsing, and unless decoding was disabled, we'll try + // decoding non UTF-8 bodies to UTF-8, using the iconv-lite library. + } else if (text_response && config.decode_response + && mime.charset && !mime.charset.match(/utf-?8$/i)) { + pipeline.push(decoder(mime.charset)); + } + + // And `out` is the stream we finally push the decoded/parsed output to. + pipeline.push(out); + + // Now release the kraken! + var tmp = resp; + while (pipeline.length) { + tmp = tmp.pipe(pipeline.shift()); + } + + // If the user has requested and output file, pipe the output stream to it. + // In stream mode, we will still get the response stream to play with. + if (config.output && resp.statusCode == 200) { + resp.pipe(fs.createWriteStream(config.output)) + } + + // Only aggregate the full body if a callback was requested. +// if (callback) { + resp.raw = []; + resp.body = []; + resp.bytes = 0; + // Count the amount of (raw) bytes passed using a PassThrough stream. + var clean_pipe = new stream.PassThrough(); + resp.pipe(clean_pipe); + + clean_pipe.on('readable', function() { + var chunk; + while (chunk = this.read()) { + resp.bytes += chunk.length; + resp.raw.push(chunk); + } + }) + + // Listen on the 'readable' event to aggregate the chunks. + out.on('readable', function() { + var chunk; + while ((chunk = this.read()) !== null) { + // We're either pushing buffers or objects, never strings. + if (typeof chunk == 'string') chunk = new Buffer(chunk); + + // Push all chunks to resp.body. We'll bind them in resp.end(). + resp.body.push(chunk); + } + }) + + // And set the .body property once all data is in. + out.on('end', function() { + + // we may want access to the raw data, so keep a reference. + resp.raw = Buffer.concat(resp.raw); + + // if parse was successful, we should have an array with one object + if (resp.body[0] !== undefined && !Buffer.isBuffer(resp.body[0])) { + resp.body = resp.body[0]; + } else { // we got one or several buffers. string or binary. + resp.body = Buffer.concat(resp.body); + + // if we're here and parsed is true, it means we tried to but it didn't work. + // so given that we got a text response, let's stringify it. + if (text_response || parsed) { + resp.body = resp.body.toString(); + } + } + // time to call back, junior. + //done(null, resp, resp.body); + + if (callback) { + done(null, resp, resp.body); + }else { + return resp.body; + } + }); + +// } + + }); // end request call + + // unless timeout was disabled, set a timeout to abort the request + if (config.timeout > 0) { + timer = setTimeout(function() { + request.abort(); + }, config.timeout) + } + + request.on('error', function(err) { + debug('Request error', err); + if (timer) clearTimeout(timer); + + done(err || new Error('Unknown error when making request.')); + }); + + if (post_data) { + if (this.is_stream(post_data)) { + post_data.pipe(request); + } else { + request.write(post_data, config.encoding); + request.end(); + } + } else { + request.end(); + } + + out.request = request; + return out; + }, + + parse_content_type: function(header) { + if (!header || header == '') return {}; + + var charset = 'iso-8859-1', arr = header.split(';'); + try { charset = arr[1].match(/charset=(.+)/)[1] } catch (e) { /* not found */ } + + return { type: arr[0], charset: charset }; + }, + + is_stream: function(obj) { + return typeof obj.pipe === 'function'; + } +} + +exports.version = version; + +exports.defaults = function(obj) { + for (var key in obj) { + if (defaults[key] && typeof obj[key] != 'undefined') + defaults[key] = obj[key]; + } + return defaults; +} + +'head get'.split(' ').forEach(function(method) { + exports[method] = function(uri, options, callback) { + return Needle.request(method, uri, null, options, callback); + } +}) + +'post put delete'.split(' ').forEach(function(method) { + exports[method] = function(uri, data, options, callback) { + return Needle.request(method, uri, data, options, callback); + } +}) + +exports.request = function(method, uri, data, opts, callback) { + return Needle.request(method, uri, data, opts, callback); +}; diff --git a/lib/needle/lib/parsers.js b/lib/needle/lib/parsers.js new file mode 100644 index 0000000..bf9c2ea --- /dev/null +++ b/lib/needle/lib/parsers.js @@ -0,0 +1,73 @@ +////////////////////////////////////////// +// Defines mappings between content-type +// and the appropriate parsers. +////////////////////////////////////////// + +var Transform = require('stream').Transform; + +function parserFactory(fn) { + + return function() { + var chunks = [], + stream = new Transform({ objectMode: true }); + + // Buffer all our data + stream._transform = function(chunk, encoding, done) { + chunks.push(chunk); + done(); + } + + // And call the parser when all is there. + stream._flush = function(done) { + var self = this, + data = Buffer.concat(chunks); + + try { + fn(data, function(err, result) { + if (err) throw err; + self.push(result); + }); + } catch (err) { + // console.error('Error while processing: ', err); + self.push(data); // just pass the original data + } finally { + done(); + } + } + + return stream; + } + +} + +module.exports['application/json'] = parserFactory(function(buffer, callback) { + + var err, data; + try { + data = JSON.parse(buffer); + } catch (e) { + err = e; + } + callback(err, data); + +}); + +module.exports['text/javascript'] = module.exports['application/json']; + +try { + + var xml2js = require('xml2js'); + + // xml2js.Parser.parseString() has the exact same function signature + // as our ParseStream expects, so we can reuse this. + module.exports['application/xml'] = parserFactory(new xml2js.Parser({ + explicitRoot : true, + explicitArray: false + }).parseString, true); + + // aliases for other XML content types + module.exports['text/xml'] = module.exports['application/xml']; + module.exports['application/rss+xml'] = module.exports['application/xml']; + module.exports['application/atom+xml'] = module.exports['application/xml']; + +} catch(e) { /* xml2js not found */ } diff --git a/lib/needle/lib/querystring.js b/lib/needle/lib/querystring.js new file mode 100644 index 0000000..ae7539e --- /dev/null +++ b/lib/needle/lib/querystring.js @@ -0,0 +1,40 @@ +// based on the qs module, but handles null objects as expected +// fixes by Tomas Pollak. + +var stringify = function(obj, prefix) { + if (obj === null || typeof obj == 'undefined') { + return prefix + '='; + } else if (obj.constructor == Array) { + return stringifyArray(obj, prefix); + } else if (typeof obj == 'object') { + return stringifyObject(obj, prefix); + } else if (prefix) { // string inside array or hash + return prefix + '=' + encodeURIComponent(String(obj)); + } else { + throw new TypeError('Object expected.'); + } +}; + +function stringifyArray(arr, prefix) { + var ret = []; + + for (var i = 0, len = arr.length; i < len; i++) { + ret.push(stringify(arr[i], prefix + '[' + i + ']')); + } + + return ret.join('&'); +} + +function stringifyObject(obj, prefix) { + var ret = []; + + Object.keys(obj).forEach(function(key) { + ret.push(stringify(obj[key], prefix + ? prefix + '[' + encodeURIComponent(key) + ']' + : encodeURIComponent(key))); + }) + + return ret.join('&'); +} + +exports.build = stringify; \ No newline at end of file diff --git a/lib/needle/package.json b/lib/needle/package.json new file mode 100644 index 0000000..b337f6d --- /dev/null +++ b/lib/needle/package.json @@ -0,0 +1,91 @@ +{ + "name": "needle", + "version": "0.7.11", + "description": "Tiny yet feature-packed HTTP client. With multipart, charset decoding and proxy support.", + "keywords": [ + "http", + "https", + "simple", + "request", + "client", + "multipart", + "upload", + "proxy", + "deflate", + "timeout", + "charset", + "iconv" + ], + "tags": [ + "http", + "https", + "simple", + "request", + "client", + "multipart", + "upload", + "proxy", + "deflate", + "timeout", + "charset", + "iconv" + ], + "author": { + "name": "Tomás Pollak", + "email": "tomas@forkhq.com" + }, + "repository": { + "type": "git", + "url": "https://github.com/tomas/needle.git" + }, + "dependencies": { + "iconv-lite": "^0.4.4" + }, + "devDependencies": { + "mocha": "", + "sinon": "", + "should": "", + "xml2js": "", + "JSONStream": "", + "q": "", + "jschardet": "" + }, + "scripts": { + "test": "mocha test" + }, + "directories": { + "lib": "./lib" + }, + "main": "./lib/needle", + "bin": { + "needle": "./bin/needle" + }, + "engines": { + "node": ">= 0.10.x" + }, + "gitHead": "d373be0c8de7188e8b8546ca6aa3acd1db55ba2c", + "bugs": { + "url": "https://github.com/tomas/needle/issues" + }, + "homepage": "https://github.com/tomas/needle", + "_id": "needle@0.7.11", + "_shasum": "2b6ef44cd4260cdc68301ad0ae496c860e13b4d8", + "_from": "needle@>=0.7.10 <0.8.0", + "_npmVersion": "1.4.28", + "_npmUser": { + "name": "tomas", + "email": "tomas@forkhq.com" + }, + "maintainers": [ + { + "name": "tomas", + "email": "tomas@forkhq.com" + } + ], + "dist": { + "shasum": "2b6ef44cd4260cdc68301ad0ae496c860e13b4d8", + "tarball": "http://registry.npmjs.org/needle/-/needle-0.7.11.tgz" + }, + "_resolved": "https://registry.npmjs.org/needle/-/needle-0.7.11.tgz", + "readme": "ERROR: No README data found!" +} diff --git a/package.json b/package.json index 82875a3..bbc4070 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,17 @@ { - "name": "iod-node", - "version": "0.1.1", - "description": "Idol OnDemand node client", + "name": "havenondemand", + "version": "1.4.0", + "description": "Official Haven OnDemand Node.js client", "main": "index.js", - "dependencies": { - "needle": "^0.7.10" - }, "devDependencies": {}, "scripts": { "test": "node test.js" }, - "author": "Martin Zerbib", + "author": "Haven OnDemand", + "contributors": [ + "Phong Vu ", + "Tyler Nappy ", + "Martin Zerbib" + ], "license": "MIT" }