javascript/httpclient
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use new URL object. Fix cookie parsing.

Browse Source
This commit is contained in:
Boudin Malin
2017-07-28 12:06:58 +02:00
parent d274ba3f96
commit 0079259c74
1 changed files with 119 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
lib/httpclient.js
View File

@@ -1,10 +1,12 @@
"use strict"; "use strict";
var http = require("http"); const http = require("http");
var https = require("https"); const https = require("https");
var url = require("url"); const { URL } = require('url');
var querystring = require("querystring"); const querystring = require("querystring");
var stream = require("stream"); const stream = require("stream");
const zlib = require('zlib');
function Cookie(rawCookie) { function Cookie(rawCookie) {
var cookieRegex = /([^,; =]+)(?:=([^;]*)(?:[ ;]+|$))/g; var cookieRegex = /([^,; =]+)(?:=([^;]*)(?:[ ;]+|$))/g;
@@ -16,23 +18,23 @@ function Cookie(rawCookie) {
var key = match[1]; var key = match[1];
var value = match[2]; var value = match[2];
switch (key) { switch (key.toLowerCase()) {
case "Path" : case "path" :
cookie.path = value; cookie.path = value;
break; break;
case "Domain" : case "domain" :
cookie.domain = value; cookie.domain = value;
break; break;
case "Expires" : case "expires" :
this.expires = Date.parse(value); cookie.expires = Date.parse(value);
break; break;
case "Max-Age" : case "max-age" :
cookie.maxAge = Number.parseInt(value); cookie.maxAge = Number.parseInt(value);
break; break;
case "Secured" : case "secured" :
cookie.secured = true; cookie.secured = true;
break; break;
case "Http-Only" : case "http-only" :
cookie.httpOnly = true; cookie.httpOnly = true;
break; break;
default : default :
@@ -55,8 +57,8 @@ var CookieJar = function () {
var currentDate = new Date(); var currentDate = new Date();
cookies.get(path).forEach( function (value, key, map) { cookies.get(path).forEach( function (value, key, map) {
if ((value.expires !== undefined && value.expires < currentDate) if (value.expires !== undefined && value.expires < currentDate
|| (value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date())) { || value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date()) {
map.delete(key); map.delete(key);
} }
else { else {
@@ -77,8 +79,8 @@ var CookieJar = function () {
cookies.set(path, new Map()); cookies.set(path, new Map());
} }
var deleteCookie = ((cookie.maxAge !== undefined && cookie.maxAge <= 0) var deleteCookie = cookie.maxAge !== undefined && cookie.maxAge <= 0
|| (cookie.expires !== undefined && cookie.expires < new Date())); || cookie.expires !== undefined && cookie.expires < new Date();
if (deleteCookie) { if (deleteCookie) {
if (cookies.get(path).has(cookie.name)) { if (cookies.get(path).has(cookie.name)) {
@@ -95,146 +97,145 @@ var CookieJar = function () {
}; };
module.exports = function Browser (_parameters) { module.exports = function Browser (parameters) {
var baseUrl = null; parameters = parameters || {};
var instance = this;
var baseUrl = parameters.baseUrl ? new URL(parameters.baseUrl) : null;
//CookieJar and custom headers //CookieJar and custom headers
var cookieJar = new CookieJar(); var cookieJar = new CookieJar();
this.headers = {}; this.headers = parameters.headers || {};
this.redirect = parameters && parameters.redirect;
if (_baseUrl) { this.ignoreErrors = parameters && parameters.ignoreErrors;
baseUrl = url.parse(_parameters.baseUrl);
}
if (_headers) {
this.headers = _parameters.headers;
}
if (this.headers["User-Agent"] === undefined) { if (this.headers["User-Agent"] === undefined) {
this.headers["User-Agent"] = "HttpClient/1.0.0 (" + os.type() + " " + os.release()+")"; this.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0";
} }
if (this.headers["Accept-Language"] === undefined) { if (this.headers["Accept-Language"] === undefined) {
this.headers["Accept-Language"] = "en-US"; this.headers["Accept-Language"] = "en,*:q=0.1";
} }
if (this.headers["Accept"] === undefined) { if (this.headers["Accept"] === undefined) {
this.headers["Accept"] = "text/plain,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; this.headers["Accept"] = "*/*";
} }
if (this.headers["Accept-Encoding"] === undefined) { if (this.headers["Accept-Encoding"] === undefined) {
this.headers["Accept-Encoding"] = "gzip,deflate"; this.headers["Accept-Encoding"] = "gzip,deflate";
} }
this.get = function (_path, _querystring) { if (this.headers["Accept-Charset"] === undefined) {
this.headers["Accept-Charset"] = "us-ascii,ISO-8859-1,ISO-8859-2,ISO-8859-3,ISO-8859-4,ISO-8859-5,ISO-8859-6,ISO-8859-7,ISO-8859-8,ISO-8859-9,ISO-8859-10,ISO-8859-13,ISO-8859-14,ISO-8859-15,ISO-8859-16,windows-1250,windows-1251,windows-1252,windows-1256,windows-1257,cp437,cp737,cp850,cp852,cp866,x-cp866-u,x-mac,x-mac-ce,x-kam-cs,koi8-r,koi8-u,koi8-ru,TCVN-5712,VISCII,utf-8";
}
if (parameters.keepAlive) {
this.headers["Connection"] = "keep-alive";
}
this.get = function (path, querystring) {
return query({ return query({
method : "GET", method : "GET",
path : _path, path: path,
querystring : _querystring, querystring: querystring
}); }, this);
}; };
this.post = function(_path, _data) { this.post = function (path, data) {
return query({ return query({
method : "POST", method : "POST",
path : _path, path: path,
data : _data data: data
}); }, this);
}; };
this.postForm = function(_path, _data) { this.postForm = function (path, data) {
return query({ return query({
method : "POST", method : "POST",
path : _path, path: path,
data : querystring.stringify(_data) data: querystring.stringify(data)
}); }, this);
}; };
this.put = function(_path, _data) { this.put = function (path, data) {
return query({ return query({
method : "PUT", method : "PUT",
path : _path, path: path,
data : _data data: data
}); }, this);
}; };
this.delete = function(_path, _querystring) { this.delete = function (path, querystring) {
return query({ return query({
method : "DELETE", method : "DELETE",
path : _path, path: path,
querystring : _querystring querystring: querystring
}); }, this);
}; };
this.patch = function(_path, _data) { this.patch = function (path, data) {
return query({ return query({
method : "PATCH", method : "PATCH",
path : _path, path: path,
data : _data data: data
}); }, this);
}; };
function query(_request) { function query(_request, client) {
var pathUrl = url.parse(_request.path); var url = new URL(_request.path);
pathUrl.headers = instance.headers; var isHttps = url.protocol === "https:";
var requestObj = {
hostname: url.hostname,
port: url.port ? url.port : isHttps ? 443 : 80,
path: url.pathname,
method: _request.method,
headers: client.headers
};
if (url.search) {
requestObj.path += url.search;
}
if (_request.querystring) { if (_request.querystring) {
let encodedQueryString = querystring.stringify(_request.querystring); let encodedQueryString = querystring.stringify(_request.querystring);
pathUrl.search = pathUrl.search ? pathUrl.search + "&" + encodedQueryString : "?" + encodedQueryString; requestObj.path += url.search ? url.search + "&" + encodedQueryString : "?" + encodedQueryString;
} }
var cookies = cookieJar.get(pathUrl.pathname); var cookies = cookieJar.get(url.pathname);
if ( pathUrl.pathname !== "/") { if (url.pathname !== "/") {
cookies += cookieJar.get("/"); cookies += cookieJar.get("/");
} }
if (cookies) { if (cookies) {
pathUrl.headers.Cookies = cookies; requestObj.headers.Cookies = cookies;
} }
if (isHttps) {
requestObj.rejectUnauthorized = false;
}
//Complete the url with the base one if a path was provided //Complete the url with the base one if a path was provided
if (baseUrl !== null && pathUrl.protocol === null) { // if (client.baseUrl !== null && pathUrl.protocol === null) {
pathUrl.protocol = baseUrl.protocol; // pathUrl.protocol = baseUrl.protocol;
pathUrl.hostname = baseUrl.hostname; // pathUrl.hostname = baseUrl.hostname;
pathUrl.auth = baseUrl.auth; // pathUrl.auth = baseUrl.auth;
pathUrl.port = baseUrl.port; // pathUrl.port = baseUrl.port;
} // }
return new Promise(function(fulfill, reject) { return new Promise((fulfill, reject) => {
var protocol; var protocol = isHttps ? https : http;
if (pathUrl.protocol === "https:") { var request = protocol.request(requestObj, function(response) {
protocol = https;
if (pathUrl.rejectUnauthorized === null || pathUrl.rejectUnauthorized === undefined) {
pathUrl.rejectUnauthorized = false;
}
if (!pathUrl.port) {
pathUrl.port = 443;
}
}
else {
protocol = http;
if (!pathUrl.port) {
pathUrl.port = 80;
}
}
pathUrl.method = _request.method;
var request = protocol.request(pathUrl, function(response) {
if (response.statusCode < 300 || (response.statusCode < 400 && !_parameters.redirect)) {
if (response.headers["set-cookie"]) { if (response.headers["set-cookie"]) {
for (let i = 0 ; i < response.headers["set-cookie"].length ; ++i) { for (let i = 0; i < response.headers["set-cookie"].length; ++i) {
cookieJar.update(response.headers["set-cookie"][i]); cookieJar.update(response.headers["set-cookie"][i]);
} }
} }
if (response.statusCode < 300 || response.statusCode < 400 && !client.redirect || response.statusCode >= 400 && client.ignoreErrors) {
let decoder; let decoder;
if (response.headers["content-encoding"] === "gzip") { if (response.headers["content-encoding"] === "gzip") {
@@ -249,10 +250,22 @@ module.exports = function Browser (_parameters) {
decoder = response; decoder = response;
} }
fulfill(response, decoder); fulfill({
response: response,
body: decoder,
url: url.toString()
});
} }
else if (response.statusCode < 400 && _parameters.redirect) { else if (response.statusCode < 400 && client.redirect) {
instance.get(response.headers.location) client.headers["Referer"] = url.protocol + "//" + url.hostname + url.pathname;
let newLocation = response.headers.location;
if (newLocation.startsWith("/")) {
newLocation = url.protocol + "//" + url.hostname + newLocation;
}
client.get(newLocation)
.then(fulfill) .then(fulfill)
.catch(reject); .catch(reject);
} }
@@ -261,7 +274,7 @@ module.exports = function Browser (_parameters) {
} }
}); });
request.on("error", function(error) { request.on("error", (error) => {
reject(error); reject(error);
}); });
@@ -269,7 +282,7 @@ module.exports = function Browser (_parameters) {
if (_request.data instanceof stream) { if (_request.data instanceof stream) {
_request.data.pipe(request); _request.data.pipe(request);
} }
else if (_request.data instanceof String || typeof(_request.data) === "string") { else if (_request.data instanceof String || typeof _request.data === "string") {
request.write(_request.data); request.write(_request.data);
} }
else { else {