diff --git a/.npmignore b/.npmignore index 0a4bf0e..6a0f3c7 100644 --- a/.npmignore +++ b/.npmignore @@ -2,3 +2,5 @@ raw/ build/ build/.* +src/*_infos.h +src/*_options.h diff --git a/README.md b/README.md index ee27a00..b25ca05 100644 --- a/README.md +++ b/README.md @@ -9,22 +9,20 @@ Quick Start * quick start curl = require('node-curl'); - curl('www.google.com', function(err, res) { - console.info(res.status); + curl('www.google.com', function(err) { + console.info(this.status); console.info('-----'); - console.info(res.body); + console.info(this.body); console.info('-----'); - console.info(res.info('SIZE_DOWNLOAD')); - res.close(); + console.info(this.info('SIZE_DOWNLOAD')); }); * with options curl = require('node-curl') - curl('www.google.com', {VERBOSE: 1, RAW: 1}, function(err, res) { - console.info(res); - res.close(); + curl('www.google.com', {VERBOSE: 1, RAW: 1}, function(err) { + console.info(this); }); Usage @@ -33,9 +31,10 @@ Usage * curl curl(url, [options = {}], callback) - callback includes 2 parameters (error, result) + callback includes 1 parameters (error) + result is stored in curl -* result in callback +* Retrieve Data from curl members: status - Http Response code @@ -44,23 +43,34 @@ Usage methods: info(name) - Get information of result, see 'info' section +* Curl Control + methods: + void reset() + - reset curl and set options to default options + + void setDefaultOptions(options, reset = true) + - set default options + + curl create(defaultOptions) + - create a new curl with default options + Options ------- -* Any Curl Easy Options +* Any cURL Easy Options eg: CURLOPT_VERBOSE will be VERBOSE, CURLOPT_HEADER will be HEADER - Full list at http://curl.haxx.se/libcurl/c/curl_easy_setopt.html + Full list at http://curl.haxx.se/libcurl/c/curl_easy_setopt.html * node-curl Extra Options - RAW - Returns Buffer instead of String in result.body + RAW - Returns Buffer instead of String in result.body * About slist parameters node-curl support slist which map to Javascript Array - eg: + eg: HTTP_HEADER: ['FOO', 'BAR'] HTTP_HEADER: 'FOO' @@ -68,7 +78,7 @@ Options Infos ----- -* Any Curl Info options +* Any cURL Info options eg: CURLINFO_EFFECTIVE_URL will be EFFETCTIVE_URL @@ -80,9 +90,6 @@ Infos slist will be returns in Array eg: CURLINFO_COOKIELIST - Hints ----- -close the result to release resource of curl immediately. -or the resource will not release until gc performed. diff --git a/examples/pressure.js b/examples/pressure.js index 25a190b..23b3377 100644 --- a/examples/pressure.js +++ b/examples/pressure.js @@ -1,30 +1,39 @@ -// Generated by ToffeeScript 1.2.0-0 +// Generated by ToffeeScript 1.3.3 (function() { - var assert, curl, i, j, next, once, _i; + var Curl, assert, i, j, next, _fn, _i; - curl = require('../index'); + Curl = require('../index'); assert = require('assert'); j = 0; (next = function() { - console.info("curl instances: ", curl.get_count()); + console.info("curl instances: ", Curl.get_count()); return setTimeout(next, 1000); })(); - for (i = _i = 1; _i <= 100; i = ++_i) { - (once = function() { + _fn = function() { + var curl, once; + curl = Curl.create(); + return (once = function() { var _this = this; return curl('localhost/test.html', function(_$$_err, _$$_res) { var err, res; err = _$$_err; res = _$$_res; assert.equal(res.body.length, 1468); - if (++j % 100 === 0) console.info(j); - if (j < 20000) return once(); + if (++j % 100 === 0) { + console.info(j); + } + if (j < 5000) { + return once(); + } }); })(); + }; + for (i = _i = 1; _i <= 100; i = ++_i) { + _fn(); } }).call(this); diff --git a/examples/pressure.toffee b/examples/pressure.toffee index f9a7cd1..db466a8 100644 --- a/examples/pressure.toffee +++ b/examples/pressure.toffee @@ -1,16 +1,18 @@ -curl = require '../index' +Curl = require '../index' assert = require 'assert' j = 0; do next = -> - console.info "curl instances: ", curl.get_count() + console.info "curl instances: ", Curl.get_count() setTimeout next, 1000 for i in [1..100] - do once = -> - err, res = curl! 'localhost/test.html' - assert.equal res.body.length, 1468 - if ++j % 100 == 0 - console.info j - if j < 20000 - once() - # res.close() + do -> + curl = Curl.create() + do once = -> + err, res = curl! 'localhost/test.html' + assert.equal res.body.length, 1468 + if ++j % 100 == 0 + console.info j + if j < 5000 + once() + # res.close() diff --git a/examples/test.js b/examples/test.js index 932e81e..146df0e 100644 --- a/examples/test.js +++ b/examples/test.js @@ -1,6 +1,6 @@ -// Generated by ToffeeScript 1.2.0-0 +// Generated by ToffeeScript 1.3.3 (function() { - var curl, fs, p, url, util, + var cookieFile, curl, fs, options, p, util, _this = this; curl = require('../index'); @@ -11,29 +11,42 @@ p = console.info; - url = 'www.google.com'; + cookieFile = 'node-curl-cookie.txt'; - curl(url, { - HTTPHEADER: 'BAR', + options = { VERBOSE: 1, - COOKIEFILE: 'node-curl-cookie.txt' - }, function(_$$_err, _$$_res) { - var err, res; + COOKIEFILE: cookieFile, + COOKIEJAR: cookieFile, + ACCEPT_ENCODING: 'gzip', + RAW: 1 + }; + + curl.debug = 1; + + curl.setDefaultOptions(options); + + curl('www.google.com', function(_$$_err) { + var err; err = _$$_err; - res = _$$_res; - p("\x1b[33m" + util.inspect(res.info('COOKIELIST')) + "\x1b[0m"); - url = 'www.yahoo.com'; - return curl(url, { - HTTPHEADER: ['foo', 'bar'], - VERBOSE: 1, - COOKIEFILE: 'node-curl-cookie.txt', - RAW: 1 - }, function(_$$_err, _$$_res) { + p("\x1b[33m" + util.inspect(curl.info('COOKIELIST')) + "\x1b[0m"); + curl.reset(); + return curl('www.yahoo.com', function(_$$_err) { + var stream; err = _$$_err; - res = _$$_res; - p("\x1b[33m" + util.inspect(res.info('COOKIELIST')) + "\x1b[0m"); - p(res.body); - return fs.unlink('node-curl-cookie.txt'); + p("\x1b[33m" + util.inspect(curl.info('COOKIELIST')) + "\x1b[0m"); + p("body length " + curl.body.length); + p("\x1b[33mText in " + cookieFile + "\x1b[0m"); + p("----"); + stream = fs.createReadStream(cookieFile); + stream.pipe(process.stdout); + return stream.on('end', function() { + var _this = this; + p("----"); + p("deleting " + cookieFile); + return fs.unlink(cookieFile, function() { + return p("done."); + }); + }); }); }); diff --git a/examples/test.toffee b/examples/test.toffee index ee7d1ef..366dfcc 100644 --- a/examples/test.toffee +++ b/examples/test.toffee @@ -3,20 +3,28 @@ fs = require 'fs' util = require 'util' p = console.info -url = 'www.google.com' -err, res = curl! url, - HTTPHEADER: 'BAR' +cookieFile = 'node-curl-cookie.txt' +options = { VERBOSE: 1 - COOKIEFILE: 'node-curl-cookie.txt' - -p "\x1b[33m" + util.inspect(res.info('COOKIELIST')) + "\x1b[0m" -url = 'www.yahoo.com' -err, res = curl! url, - HTTPHEADER: ['foo', 'bar'] - VERBOSE: 1 - COOKIEFILE: 'node-curl-cookie.txt' + COOKIEFILE: cookieFile + COOKIEJAR: cookieFile + ACCEPT_ENCODING: 'gzip' RAW: 1 - -p "\x1b[33m" + util.inspect(res.info('COOKIELIST')) + "\x1b[0m" -p res.body -fs.unlink 'node-curl-cookie.txt' +} +curl.debug = 1 +curl.setDefaultOptions options +err = curl! 'www.google.com' +p "\x1b[33m" + util.inspect(curl.info('COOKIELIST')) + "\x1b[0m" +curl.reset() +err = curl! 'www.yahoo.com' +p "\x1b[33m" + util.inspect(curl.info('COOKIELIST')) + "\x1b[0m" +p "body length #{curl.body.length}" +p "\x1b[33mText in #{cookieFile}\x1b[0m" +p "----" +stream = fs.createReadStream(cookieFile) +stream.pipe(process.stdout) +stream.on 'end', -> + p "----" + p "deleting #{cookieFile}" + fs.unlink! cookieFile + p "done." diff --git a/lib/CurlBuilder.js b/lib/CurlBuilder.js new file mode 100644 index 0000000..8a037d5 --- /dev/null +++ b/lib/CurlBuilder.js @@ -0,0 +1,193 @@ +// Generated by ToffeeScript 1.3.3 +(function() { + var Curl, CurlBuilder, + __hasProp = {}.hasOwnProperty, + __slice = [].slice; + + try { + Curl = require(__dirname + '/../build/Release/node-curl').Curl; + } catch (e) { + Curl = require(__dirname + '/../build/default/node-curl').Curl; + } + + CurlBuilder = (function() { + + function CurlBuilder() {} + + CurlBuilder.curls = {}; + + CurlBuilder.id = 0; + + CurlBuilder.close_all = function() { + var curl, id, _ref; + _ref = CurlBuilder.curls; + for (id in _ref) { + if (!__hasProp.call(_ref, id)) continue; + curl = _ref[id]; + curl.end(); + delete CurlBuilder.curls[id]; + } + return CurlBuilder; + }; + + CurlBuilder.create = function(defaultOptions) { + var curl; + curl = function() { + return curl.perform.apply(curl, arguments); + }; + curl.perform = function() { + var args, c, cb, length, options, url; + args = 1 <= arguments.length ? __slice.call(arguments, 0) : []; + if (this.running) { + throw new Error('the cURL session is busy, use CurlBuilder.create to create another cURL Session'); + } + if (!this.curl_) { + throw new Error('the cURL is closed.'); + } + this.running = true; + cb = args.pop(); + url = args[0], options = args[1]; + if (options == null) { + options = {}; + } + c = this.curl_; + options['URL'] = url; + c.chunks = []; + length = 0; + this.setOptions(options); + c.on_write = function(chunk) { + curl.log("receive " + chunk.length + " bytes"); + c.chunks.push(chunk); + return length += chunk.length; + }; + c.on_end = function() { + var chunk, data, position, _i, _len, _ref, + _this = this; + curl.log("receive succeeded."); + curl.running = false; + data = new Buffer(length); + position = 0; + _ref = c.chunks; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + chunk = _ref[_i]; + chunk.copy(data, position); + position += chunk.length; + } + c.chunks = []; + if (c.options.RAW) { + curl.body = data; + } else { + curl.body = data.toString(); + } + curl.status = curl.code = c.getinfo('RESPONSE_CODE'); + return process.nextTick(function() { + return cb.call(curl, null, curl); + }); + }; + c.on_error = function(err) { + var _this = this; + curl.log("receive failed: " + err.message); + curl.running = false; + return process.nextTick(function() { + return cb.call(curl, err, null); + }); + }; + this.log('perform'); + return c.perform(); + }; + curl.setDefaultOptions = function(options, reset) { + if (options == null) { + options = {}; + } + if (reset == null) { + reset = true; + } + defaultOptions = options; + if (reset) { + this.log('Set default options and reset cURL'); + return this.reset(); + } + }; + curl.log = function(text) { + if (this.debug) { + return console.info(("[cURL " + this.id + "] ") + text); + } + }; + curl.setOptions = function(options) { + var k, v; + if (options == null) { + options = {}; + } + for (k in options) { + if (!__hasProp.call(options, k)) continue; + v = options[k]; + this.log("Set option '" + k + "' to '" + v + "'"); + this.curl_.setopt(k, v); + } + return this; + }; + curl.setopts = function(options) { + if (options == null) { + options = {}; + } + return this.setOptions(options); + }; + curl.info = function(info) { + if (this.curl_ == null) { + throw new Error('curl is closed'); + } + return this.curl_.getinfo(info); + }; + curl.end = function() { + if (this.curl_ != null) { + this.curl_.close(); + } + this.curl_ = null; + this.body = null; + delete CurlBuilder.curls[this.id]; + return this.log("closed."); + }; + curl.close = function() { + return this.end(); + }; + curl.open = function() { + if (curl.id == null) { + curl.id = ++CurlBuilder.id; + } + this.log("opening."); + this.curl_ = new Curl(); + this.curl_.options = {}; + this.setOptions(defaultOptions); + CurlBuilder.curls[curl.id] = curl; + return this.log("opened."); + }; + curl.reset = function() { + this.log('reset'); + if (this.curl_) { + this.end(); + } + return this.open(); + }; + curl.Curl = Curl; + curl.Builder = CurlBuilder; + curl.create = function(defaultOptions) { + return CurlBuilder.create(defaultOptions); + }; + curl.get_count = function() { + return Curl.get_count(); + }; + curl.open(); + return curl; + }; + + return CurlBuilder; + + }).call(this); + + process.on('exit', function() { + return CurlBuilder.close_all(); + }); + + module.exports = CurlBuilder; + +}).call(this); diff --git a/lib/CurlBuilder.toffee b/lib/CurlBuilder.toffee new file mode 100644 index 0000000..b6267a5 --- /dev/null +++ b/lib/CurlBuilder.toffee @@ -0,0 +1,136 @@ +try + {Curl} = require __dirname + '/../build/Release/node-curl' +catch e + {Curl} = require __dirname + '/../build/default/node-curl' + +class CurlBuilder + @curls: {} + @id: 0 + @close_all: => + for own id, curl of @curls + curl.end() + delete @curls[id] + @ + + @create: (defaultOptions) => + curl = -> + curl.perform.apply curl, arguments + + curl.perform = (args...) -> + if @running + throw new Error 'the cURL session is busy, use CurlBuilder.create to create another cURL Session' + + if !@curl_ + throw new Error 'the cURL is closed.' + + @running = true + cb = args.pop() + [url, options] = args + options ?= {} + + c = @curl_ + options['URL'] = url + c.chunks = [] + length = 0 + + @setOptions options + + c.on_write = (chunk) -> + curl.log "receive #{chunk.length} bytes" + c.chunks.push chunk + length += chunk.length + + c.on_end = -> + curl.log "receive succeeded." + curl.running = false + data = new Buffer(length) + position = 0 + for chunk in c.chunks + chunk.copy data, position + position += chunk.length + c.chunks = [] + + if c.options.RAW + curl.body = data + else + curl.body = data.toString() + curl.status = curl.code = c.getinfo('RESPONSE_CODE') + + # if curl returns to fast, avoid cb recursive call + process.nextTick! + cb.call curl, null, curl + + c.on_error = (err)-> + curl.log "receive failed: #{err.message}" + curl.running = false + process.nextTick! + cb.call curl, err, null + + @log 'perform' + c.perform() + + + + curl.setDefaultOptions = (options = {}, reset = true) -> + defaultOptions = options + if reset + @log 'Set default options and reset cURL' + @reset() + + curl.log = (text) -> + if @debug + console.info "[cURL #{@id}] " + text + curl.setOptions = (options = {}) -> + for own k, v of options + @log "Set option '#{k}' to '#{v}'" + @curl_.setopt k, v + @ + + curl.setopts = (options = {}) -> + @setOptions options + + curl.info = (info)-> + unless @curl_? + throw new Error('curl is closed') + @curl_.getinfo(info) + + curl.end = -> + @curl_.close() if @curl_? + @curl_ = null + @body = null + delete CurlBuilder.curls[@id] + @log "closed." + + curl.close = -> + @end() + + curl.open = -> + unless curl.id? + curl.id = ++CurlBuilder.id + @log "opening." + @curl_ = new Curl() + @curl_.options = {} + @setOptions defaultOptions + CurlBuilder.curls[curl.id] = curl + @log "opened." + + curl.reset = -> + @log 'reset' + if @curl_ + @end() + @open() + + curl.Curl = Curl + curl.Builder = CurlBuilder + curl.create = (defaultOptions) -> + CurlBuilder.create(defaultOptions) + curl.get_count = -> + Curl.get_count() + + curl.open() + curl + +process.on 'exit', -> + CurlBuilder.close_all() + +module.exports = CurlBuilder diff --git a/lib/curl.js b/lib/curl.js index d92125d..3573183 100644 --- a/lib/curl.js +++ b/lib/curl.js @@ -1,8 +1,6 @@ -// Generated by ToffeeScript 1.2.0-0 +// Generated by ToffeeScript 1.3.3 (function() { - var Curl, curl, curl_id, curls, - __slice = [].slice, - __hasProp = {}.hasOwnProperty; + var Curl, CurlBuilder, curl; try { Curl = require(__dirname + '/../build/Release/node-curl').Curl; @@ -10,6 +8,8 @@ Curl = require(__dirname + '/../build/default/node-curl').Curl; } + CurlBuilder = require(__dirname + '/CurlBuilder'); + Curl.prototype.setopt_user_ = function(option_id, value) { return this.options[option_id] = value; }; @@ -46,18 +46,20 @@ } }; + Curl.user_options = { + RAW: 'RAW' + }; + Curl.prototype.perform = function() { this.perform_(); return Curl.process(); }; - Curl.user_options = { - RAW: 'RAW' - }; - Curl.process = function() { var once; - if (Curl.in_process) return; + if (Curl.in_process) { + return; + } return (once = function() { var num; num = Curl.process_(); @@ -70,85 +72,7 @@ })(); }; - curl_id = 0; - - curls = {}; - - curl = function() { - var args, c, cb, k, length, options, url, v; - args = 1 <= arguments.length ? __slice.call(arguments, 0) : []; - cb = args.pop(); - url = args[0], options = args[1]; - if (options == null) options = {}; - c = new Curl(); - c.options = {}; - c.id = ++curl_id; - curls[c.id] = c; - c.setopt('FOLLOWLOCATION', 1); - c.setopt('ACCEPT_ENCODING', 'gzip'); - c.chunks = []; - length = 0; - for (k in options) { - if (!__hasProp.call(options, k)) continue; - v = options[k]; - c.setopt(k, v); - } - c.on_write = function(chunk) { - c.chunks.push(chunk); - return length += chunk.length; - }; - c.on_end = function() { - var chunk, data, position, res, _i, _len, _ref, - _this = this; - data = new Buffer(length); - position = 0; - _ref = c.chunks; - for (_i = 0, _len = _ref.length; _i < _len; _i++) { - chunk = _ref[_i]; - chunk.copy(data, position); - position += chunk.length; - } - c.chunks = []; - res = {}; - res.curl_ = c; - delete curls[c.id]; - if (c.options.RAW) { - res.body = data; - } else { - res.body = data.toString(); - } - res.status = res.code = c.getinfo('RESPONSE_CODE'); - res.info = function(info) { - if (this.curl_ == null) throw new Error('curl is closed'); - return this.curl_.getinfo(info); - }; - res.close = function() { - if (this.curl_ != null) this.curl_.close(); - this.curl_ = null; - return this.body = null; - }; - return process.nextTick(function() { - return cb(null, res); - }); - }; - c.on_error = function(err) { - var _this = this; - curls[c.id].close(); - delete curls[c.id]; - return process.nextTick(function() { - return cb(err, null); - }); - }; - c.setopt('URL', url); - c.perform(); - return c; - }; - - curl.Curl = Curl; - - curl.get_count = function() { - return Curl.get_count(); - }; + curl = CurlBuilder.create(); module.exports = curl; diff --git a/lib/curl.toffee b/lib/curl.toffee index 782863f..b109289 100644 --- a/lib/curl.toffee +++ b/lib/curl.toffee @@ -3,6 +3,8 @@ try catch e {Curl} = require __dirname + '/../build/default/node-curl' +CurlBuilder = require __dirname + '/CurlBuilder' + Curl::setopt_user_ = (option_id, value) -> @options[option_id] = value @@ -35,13 +37,13 @@ Curl::getinfo = (oinfo) -> else throw new Error("unsupported info #{oinfo}") +Curl.user_options = + RAW: 'RAW' + Curl::perform = -> @perform_() Curl.process() -Curl.user_options = - RAW: 'RAW' - Curl.process = -> if Curl.in_process return @@ -53,77 +55,5 @@ Curl.process = -> else Curl.in_process = false -# url, [options], cb -curl_id = 0 -curls = {} -curl = (args...) -> - cb = args.pop() - [url, options] = args - options ?= {} - - c = new Curl() - c.options = {} - c.id = ++curl_id - # after the scope c will not valid any more, so add to curls to keep c alive - curls[c.id] = c - - c.setopt 'FOLLOWLOCATION', 1 - c.setopt 'ACCEPT_ENCODING', 'gzip' - c.chunks = [] - length = 0 - - for own k, v of options - c.setopt k, v - - c.on_write = (chunk) -> - c.chunks.push chunk - length += chunk.length - - c.on_end = -> - data = new Buffer(length) - position = 0 - for chunk in c.chunks - chunk.copy data, position - position += chunk.length - c.chunks = [] - - res = {} - # now the c is in res.curl, delete curl in curls - # if res destruct, curl will be destruct after gc - res.curl_ = c - delete curls[c.id] - - if c.options.RAW - res.body = data - else - res.body = data.toString() #.toString() - res.status = res.code = c.getinfo('RESPONSE_CODE') - - res.info = (info)-> - unless @curl_? - throw new Error('curl is closed') - @curl_.getinfo(info) - - res.close = -> - @curl_.close() if @curl_? - @curl_ = null - @body = null - - # if curl returns to fast, avoid cb recursive call - process.nextTick! - cb null, res - - c.on_error = (err)-> - curls[c.id].close() - delete curls[c.id] - process.nextTick! - cb err, null - - c.setopt('URL', url) - c.perform() - c - -curl.Curl = Curl -curl.get_count = -> - Curl.get_count() +curl = CurlBuilder.create() module.exports = curl diff --git a/src/node-curl.h b/src/node-curl.h index b16d54f..f194252 100644 --- a/src/node-curl.h +++ b/src/node-curl.h @@ -187,7 +187,7 @@ class NodeCurl cur = slist; while (cur) { - array->Set(array->Length(), v8::String::New(slist->data)); + array->Set(array->Length(), v8::String::New(cur->data)); cur = cur->next; } curl_slist_free_all(slist);