Use new URL object. Fix cookie parsing.
This commit is contained in:
Boudin Malin
@@ -1,10 +1,12 @@
|
||||
"use strict";
|
||||
|
||||
var http = require("http");
|
||||
var https = require("https");
|
||||
var url = require("url");
|
||||
var querystring = require("querystring");
|
||||
var stream = require("stream");
|
||||
const http = require("http");
|
||||
const https = require("https");
|
||||
const { URL } = require('url');
|
||||
const querystring = require("querystring");
|
||||
const stream = require("stream");
|
||||
const zlib = require('zlib');
|
||||
|
||||
|
||||
function Cookie(rawCookie) {
|
||||
var cookieRegex = /([^,; =]+)(?:=([^;]*)(?:[ ;]+|$))/g;
|
||||
@@ -16,23 +18,23 @@ function Cookie(rawCookie) {
|
||||
var key = match[1];
|
||||
var value = match[2];
|
||||
|
||||
switch (key) {
|
||||
case "Path" :
|
||||
switch (key.toLowerCase()) {
|
||||
case "path" :
|
||||
cookie.path = value;
|
||||
break;
|
||||
case "Domain" :
|
||||
case "domain" :
|
||||
cookie.domain = value;
|
||||
break;
|
||||
case "Expires" :
|
||||
this.expires = Date.parse(value);
|
||||
case "expires" :
|
||||
cookie.expires = Date.parse(value);
|
||||
break;
|
||||
case "Max-Age" :
|
||||
case "max-age" :
|
||||
cookie.maxAge = Number.parseInt(value);
|
||||
break;
|
||||
case "Secured" :
|
||||
case "secured" :
|
||||
cookie.secured = true;
|
||||
break;
|
||||
case "Http-Only" :
|
||||
case "http-only" :
|
||||
cookie.httpOnly = true;
|
||||
break;
|
||||
default :
|
||||
@@ -55,8 +57,8 @@ var CookieJar = function () {
|
||||
var currentDate = new Date();
|
||||
|
||||
cookies.get(path).forEach( function (value, key, map) {
|
||||
if ((value.expires !== undefined && value.expires < currentDate)
|
||||
|| (value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date())) {
|
||||
if (value.expires !== undefined && value.expires < currentDate
|
||||
|| value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date()) {
|
||||
map.delete(key);
|
||||
}
|
||||
else {
|
||||
@@ -77,8 +79,8 @@ var CookieJar = function () {
|
||||
cookies.set(path, new Map());
|
||||
}
|
||||
|
||||
var deleteCookie = ((cookie.maxAge !== undefined && cookie.maxAge <= 0)
|
||||
|| (cookie.expires !== undefined && cookie.expires < new Date()));
|
||||
var deleteCookie = cookie.maxAge !== undefined && cookie.maxAge <= 0
|
||||
|| cookie.expires !== undefined && cookie.expires < new Date();
|
||||
|
||||
if (deleteCookie) {
|
||||
if (cookies.get(path).has(cookie.name)) {
|
||||
@@ -95,146 +97,145 @@ var CookieJar = function () {
|
||||
};
|
||||
|
||||
|
||||
module.exports = function Browser (_parameters) {
|
||||
var baseUrl = null;
|
||||
var instance = this;
|
||||
module.exports = function Browser (parameters) {
|
||||
parameters = parameters || {};
|
||||
|
||||
var baseUrl = parameters.baseUrl ? new URL(parameters.baseUrl) : null;
|
||||
|
||||
//CookieJar and custom headers
|
||||
var cookieJar = new CookieJar();
|
||||
this.headers = {};
|
||||
|
||||
if (_baseUrl) {
|
||||
baseUrl = url.parse(_parameters.baseUrl);
|
||||
}
|
||||
|
||||
if (_headers) {
|
||||
this.headers = _parameters.headers;
|
||||
}
|
||||
|
||||
this.headers = parameters.headers || {};
|
||||
this.redirect = parameters && parameters.redirect;
|
||||
this.ignoreErrors = parameters && parameters.ignoreErrors;
|
||||
|
||||
if (this.headers["User-Agent"] === undefined) {
|
||||
this.headers["User-Agent"] = "HttpClient/1.0.0 (" + os.type() + " " + os.release()+")";
|
||||
this.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0";
|
||||
}
|
||||
|
||||
if (this.headers["Accept-Language"] === undefined) {
|
||||
this.headers["Accept-Language"] = "en-US";
|
||||
this.headers["Accept-Language"] = "en,*:q=0.1";
|
||||
}
|
||||
|
||||
if (this.headers["Accept"] === undefined) {
|
||||
this.headers["Accept"] = "text/plain,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
||||
this.headers["Accept"] = "*/*";
|
||||
}
|
||||
|
||||
if (this.headers["Accept-Encoding"] === undefined) {
|
||||
this.headers["Accept-Encoding"] = "gzip,deflate";
|
||||
}
|
||||
|
||||
this.get = function (_path, _querystring) {
|
||||
if (this.headers["Accept-Charset"] === undefined) {
|
||||
this.headers["Accept-Charset"] = "us-ascii,ISO-8859-1,ISO-8859-2,ISO-8859-3,ISO-8859-4,ISO-8859-5,ISO-8859-6,ISO-8859-7,ISO-8859-8,ISO-8859-9,ISO-8859-10,ISO-8859-13,ISO-8859-14,ISO-8859-15,ISO-8859-16,windows-1250,windows-1251,windows-1252,windows-1256,windows-1257,cp437,cp737,cp850,cp852,cp866,x-cp866-u,x-mac,x-mac-ce,x-kam-cs,koi8-r,koi8-u,koi8-ru,TCVN-5712,VISCII,utf-8";
|
||||
}
|
||||
|
||||
if (parameters.keepAlive) {
|
||||
this.headers["Connection"] = "keep-alive";
|
||||
}
|
||||
|
||||
this.get = function (path, querystring) {
|
||||
return query({
|
||||
method : "GET",
|
||||
path : _path,
|
||||
querystring : _querystring,
|
||||
});
|
||||
path: path,
|
||||
querystring: querystring
|
||||
}, this);
|
||||
};
|
||||
|
||||
this.post = function(_path, _data) {
|
||||
this.post = function (path, data) {
|
||||
return query({
|
||||
method : "POST",
|
||||
path : _path,
|
||||
data : _data
|
||||
});
|
||||
path: path,
|
||||
data: data
|
||||
}, this);
|
||||
};
|
||||
|
||||
this.postForm = function(_path, _data) {
|
||||
this.postForm = function (path, data) {
|
||||
return query({
|
||||
method : "POST",
|
||||
path : _path,
|
||||
data : querystring.stringify(_data)
|
||||
});
|
||||
path: path,
|
||||
data: querystring.stringify(data)
|
||||
}, this);
|
||||
};
|
||||
|
||||
this.put = function(_path, _data) {
|
||||
this.put = function (path, data) {
|
||||
return query({
|
||||
method : "PUT",
|
||||
path : _path,
|
||||
data : _data
|
||||
});
|
||||
path: path,
|
||||
data: data
|
||||
}, this);
|
||||
};
|
||||
|
||||
this.delete = function(_path, _querystring) {
|
||||
this.delete = function (path, querystring) {
|
||||
return query({
|
||||
method : "DELETE",
|
||||
path : _path,
|
||||
querystring : _querystring
|
||||
});
|
||||
path: path,
|
||||
querystring: querystring
|
||||
}, this);
|
||||
};
|
||||
|
||||
this.patch = function(_path, _data) {
|
||||
this.patch = function (path, data) {
|
||||
return query({
|
||||
method : "PATCH",
|
||||
path : _path,
|
||||
data : _data
|
||||
});
|
||||
path: path,
|
||||
data: data
|
||||
}, this);
|
||||
};
|
||||
|
||||
function query(_request) {
|
||||
var pathUrl = url.parse(_request.path);
|
||||
|
||||
pathUrl.headers = instance.headers;
|
||||
function query(_request, client) {
|
||||
var url = new URL(_request.path);
|
||||
|
||||
var isHttps = url.protocol === "https:";
|
||||
|
||||
var requestObj = {
|
||||
hostname: url.hostname,
|
||||
port: url.port ? url.port : isHttps ? 443 : 80,
|
||||
path: url.pathname,
|
||||
method: _request.method,
|
||||
headers: client.headers
|
||||
};
|
||||
|
||||
if (url.search) {
|
||||
requestObj.path += url.search;
|
||||
}
|
||||
|
||||
if (_request.querystring) {
|
||||
let encodedQueryString = querystring.stringify(_request.querystring);
|
||||
pathUrl.search = pathUrl.search ? pathUrl.search + "&" + encodedQueryString : "?" + encodedQueryString;
|
||||
requestObj.path += url.search ? url.search + "&" + encodedQueryString : "?" + encodedQueryString;
|
||||
}
|
||||
|
||||
var cookies = cookieJar.get(pathUrl.pathname);
|
||||
var cookies = cookieJar.get(url.pathname);
|
||||
|
||||
if ( pathUrl.pathname !== "/") {
|
||||
if (url.pathname !== "/") {
|
||||
cookies += cookieJar.get("/");
|
||||
}
|
||||
|
||||
if (cookies) {
|
||||
pathUrl.headers.Cookies = cookies;
|
||||
requestObj.headers.Cookies = cookies;
|
||||
}
|
||||
|
||||
if (isHttps) {
|
||||
requestObj.rejectUnauthorized = false;
|
||||
}
|
||||
|
||||
|
||||
//Complete the url with the base one if a path was provided
|
||||
if (baseUrl !== null && pathUrl.protocol === null) {
|
||||
pathUrl.protocol = baseUrl.protocol;
|
||||
pathUrl.hostname = baseUrl.hostname;
|
||||
pathUrl.auth = baseUrl.auth;
|
||||
pathUrl.port = baseUrl.port;
|
||||
}
|
||||
// if (client.baseUrl !== null && pathUrl.protocol === null) {
|
||||
// pathUrl.protocol = baseUrl.protocol;
|
||||
// pathUrl.hostname = baseUrl.hostname;
|
||||
// pathUrl.auth = baseUrl.auth;
|
||||
// pathUrl.port = baseUrl.port;
|
||||
// }
|
||||
|
||||
return new Promise(function(fulfill, reject) {
|
||||
var protocol;
|
||||
|
||||
if (pathUrl.protocol === "https:") {
|
||||
protocol = https;
|
||||
|
||||
if (pathUrl.rejectUnauthorized === null || pathUrl.rejectUnauthorized === undefined) {
|
||||
pathUrl.rejectUnauthorized = false;
|
||||
}
|
||||
|
||||
if (!pathUrl.port) {
|
||||
pathUrl.port = 443;
|
||||
}
|
||||
}
|
||||
else {
|
||||
protocol = http;
|
||||
|
||||
if (!pathUrl.port) {
|
||||
pathUrl.port = 80;
|
||||
}
|
||||
}
|
||||
|
||||
pathUrl.method = _request.method;
|
||||
return new Promise((fulfill, reject) => {
|
||||
var protocol = isHttps ? https : http;
|
||||
|
||||
var request = protocol.request(pathUrl, function(response) {
|
||||
if (response.statusCode < 300 || (response.statusCode < 400 && !_parameters.redirect)) {
|
||||
if (response.headers["set-cookie"]) {
|
||||
for (let i = 0 ; i < response.headers["set-cookie"].length ; ++i) {
|
||||
cookieJar.update(response.headers["set-cookie"][i]);
|
||||
}
|
||||
var request = protocol.request(requestObj, function(response) {
|
||||
if (response.headers["set-cookie"]) {
|
||||
for (let i = 0; i < response.headers["set-cookie"].length; ++i) {
|
||||
cookieJar.update(response.headers["set-cookie"][i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (response.statusCode < 300 || response.statusCode < 400 && !client.redirect || response.statusCode >= 400 && client.ignoreErrors) {
|
||||
let decoder;
|
||||
|
||||
if (response.headers["content-encoding"] === "gzip") {
|
||||
@@ -249,10 +250,22 @@ module.exports = function Browser (_parameters) {
|
||||
decoder = response;
|
||||
}
|
||||
|
||||
fulfill(response, decoder);
|
||||
fulfill({
|
||||
response: response,
|
||||
body: decoder,
|
||||
url: url.toString()
|
||||
});
|
||||
}
|
||||
else if (response.statusCode < 400 && _parameters.redirect) {
|
||||
instance.get(response.headers.location)
|
||||
else if (response.statusCode < 400 && client.redirect) {
|
||||
client.headers["Referer"] = url.protocol + "//" + url.hostname + url.pathname;
|
||||
|
||||
let newLocation = response.headers.location;
|
||||
|
||||
if (newLocation.startsWith("/")) {
|
||||
newLocation = url.protocol + "//" + url.hostname + newLocation;
|
||||
}
|
||||
|
||||
client.get(newLocation)
|
||||
.then(fulfill)
|
||||
.catch(reject);
|
||||
}
|
||||
@@ -261,15 +274,15 @@ module.exports = function Browser (_parameters) {
|
||||
}
|
||||
});
|
||||
|
||||
request.on("error", function(error) {
|
||||
request.on("error", (error) => {
|
||||
reject(error);
|
||||
});
|
||||
|
||||
|
||||
if (_request.data) {
|
||||
if (_request.data instanceof stream) {
|
||||
_request.data.pipe(request);
|
||||
}
|
||||
else if (_request.data instanceof String || typeof(_request.data) === "string") {
|
||||
else if (_request.data instanceof String || typeof _request.data === "string") {
|
||||
request.write(_request.data);
|
||||
}
|
||||
else {
|
||||
|
||||
Reference in New Issue
Block a user