javascript/httpclient
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use new URL object. Fix cookie parsing.

Browse Source
This commit is contained in:
Boudin Malin
2017-07-28 12:06:58 +02:00
parent d274ba3f96
commit 0079259c74
1 changed files with 119 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
lib/httpclient.js
View File

@@ -1,10 +1,12 @@
"use strict";
var http = require("http");
var https = require("https");
var url = require("url");
var querystring = require("querystring");
var stream = require("stream");
const http = require("http");
const https = require("https");
const { URL } = require('url');
const querystring = require("querystring");
const stream = require("stream");
const zlib = require('zlib');
function Cookie(rawCookie) {
var cookieRegex = /([^,; =]+)(?:=([^;]*)(?:[ ;]+|$))/g;
@@ -16,23 +18,23 @@ function Cookie(rawCookie) {
var key = match[1];
var value = match[2];
switch (key) {
case "Path" :
switch (key.toLowerCase()) {
case "path" :
cookie.path = value;
break;
case "Domain" :
case "domain" :
cookie.domain = value;
break;
case "Expires" :
this.expires = Date.parse(value);
case "expires" :
cookie.expires = Date.parse(value);
break;
case "Max-Age" :
case "max-age" :
cookie.maxAge = Number.parseInt(value);
break;
case "Secured" :
case "secured" :
cookie.secured = true;
break;
case "Http-Only" :
case "http-only" :
cookie.httpOnly = true;
break;
default :
@@ -55,8 +57,8 @@ var CookieJar = function () {
var currentDate = new Date();
cookies.get(path).forEach( function (value, key, map) {
if ((value.expires !== undefined && value.expires < currentDate)
|| (value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date())) {
if (value.expires !== undefined && value.expires < currentDate
|| value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date()) {
map.delete(key);
}
else {
@@ -77,8 +79,8 @@ var CookieJar = function () {
cookies.set(path, new Map());
}
var deleteCookie = ((cookie.maxAge !== undefined && cookie.maxAge <= 0)
|| (cookie.expires !== undefined && cookie.expires < new Date()));
var deleteCookie = cookie.maxAge !== undefined && cookie.maxAge <= 0
|| cookie.expires !== undefined && cookie.expires < new Date();
if (deleteCookie) {
if (cookies.get(path).has(cookie.name)) {
@@ -95,146 +97,145 @@ var CookieJar = function () {
};
module.exports = function Browser (_parameters) {
var baseUrl = null;
var instance = this;
module.exports = function Browser (parameters) {
parameters = parameters || {};
var baseUrl = parameters.baseUrl ? new URL(parameters.baseUrl) : null;
//CookieJar and custom headers
var cookieJar = new CookieJar();
this.headers = {};
if (_baseUrl) {
baseUrl = url.parse(_parameters.baseUrl);
}
if (_headers) {
this.headers = _parameters.headers;
}
this.headers = parameters.headers || {};
this.redirect = parameters && parameters.redirect;
this.ignoreErrors = parameters && parameters.ignoreErrors;
if (this.headers["User-Agent"] === undefined) {
this.headers["User-Agent"] = "HttpClient/1.0.0 (" + os.type() + " " + os.release()+")";
this.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0";
}
if (this.headers["Accept-Language"] === undefined) {
this.headers["Accept-Language"] = "en-US";
this.headers["Accept-Language"] = "en,*:q=0.1";
}
if (this.headers["Accept"] === undefined) {
this.headers["Accept"] = "text/plain,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
this.headers["Accept"] = "*/*";
}
if (this.headers["Accept-Encoding"] === undefined) {
this.headers["Accept-Encoding"] = "gzip,deflate";
}
this.get = function (_path, _querystring) {
if (this.headers["Accept-Charset"] === undefined) {
this.headers["Accept-Charset"] = "us-ascii,ISO-8859-1,ISO-8859-2,ISO-8859-3,ISO-8859-4,ISO-8859-5,ISO-8859-6,ISO-8859-7,ISO-8859-8,ISO-8859-9,ISO-8859-10,ISO-8859-13,ISO-8859-14,ISO-8859-15,ISO-8859-16,windows-1250,windows-1251,windows-1252,windows-1256,windows-1257,cp437,cp737,cp850,cp852,cp866,x-cp866-u,x-mac,x-mac-ce,x-kam-cs,koi8-r,koi8-u,koi8-ru,TCVN-5712,VISCII,utf-8";
}
if (parameters.keepAlive) {
this.headers["Connection"] = "keep-alive";
}
this.get = function (path, querystring) {
return query({
method : "GET",
path : _path,
querystring : _querystring,
});
path: path,
querystring: querystring
}, this);
};
this.post = function(_path, _data) {
this.post = function (path, data) {
return query({
method : "POST",
path : _path,
data : _data
});
path: path,
data: data
}, this);
};
this.postForm = function(_path, _data) {
this.postForm = function (path, data) {
return query({
method : "POST",
path : _path,
data : querystring.stringify(_data)
});
path: path,
data: querystring.stringify(data)
}, this);
};
this.put = function(_path, _data) {
this.put = function (path, data) {
return query({
method : "PUT",
path : _path,
data : _data
});
path: path,
data: data
}, this);
};
this.delete = function(_path, _querystring) {
this.delete = function (path, querystring) {
return query({
method : "DELETE",
path : _path,
querystring : _querystring
});
path: path,
querystring: querystring
}, this);
};
this.patch = function(_path, _data) {
this.patch = function (path, data) {
return query({
method : "PATCH",
path : _path,
data : _data
});
path: path,
data: data
}, this);
};
function query(_request) {
var pathUrl = url.parse(_request.path);
function query(_request, client) {
var url = new URL(_request.path);
pathUrl.headers = instance.headers;
var isHttps = url.protocol === "https:";
var requestObj = {
hostname: url.hostname,
port: url.port ? url.port : isHttps ? 443 : 80,
path: url.pathname,
method: _request.method,
headers: client.headers
};
if (url.search) {
requestObj.path += url.search;
}
if (_request.querystring) {
let encodedQueryString = querystring.stringify(_request.querystring);
pathUrl.search = pathUrl.search ? pathUrl.search + "&" + encodedQueryString : "?" + encodedQueryString;
requestObj.path += url.search ? url.search + "&" + encodedQueryString : "?" + encodedQueryString;
}
var cookies = cookieJar.get(pathUrl.pathname);
var cookies = cookieJar.get(url.pathname);
if ( pathUrl.pathname !== "/") {
if (url.pathname !== "/") {
cookies += cookieJar.get("/");
}
if (cookies) {
pathUrl.headers.Cookies = cookies;
requestObj.headers.Cookies = cookies;
}
if (isHttps) {
requestObj.rejectUnauthorized = false;
}
//Complete the url with the base one if a path was provided
if (baseUrl !== null && pathUrl.protocol === null) {
pathUrl.protocol = baseUrl.protocol;
pathUrl.hostname = baseUrl.hostname;
pathUrl.auth = baseUrl.auth;
pathUrl.port = baseUrl.port;
}
// if (client.baseUrl !== null && pathUrl.protocol === null) {
// pathUrl.protocol = baseUrl.protocol;
// pathUrl.hostname = baseUrl.hostname;
// pathUrl.auth = baseUrl.auth;
// pathUrl.port = baseUrl.port;
// }
return new Promise(function(fulfill, reject) {
var protocol;
return new Promise((fulfill, reject) => {
var protocol = isHttps ? https : http;
if (pathUrl.protocol === "https:") {
protocol = https;
if (pathUrl.rejectUnauthorized === null || pathUrl.rejectUnauthorized === undefined) {
pathUrl.rejectUnauthorized = false;
}
if (!pathUrl.port) {
pathUrl.port = 443;
}
}
else {
protocol = http;
if (!pathUrl.port) {
pathUrl.port = 80;
}
}
pathUrl.method = _request.method;
var request = protocol.request(pathUrl, function(response) {
if (response.statusCode < 300 || (response.statusCode < 400 && !_parameters.redirect)) {
var request = protocol.request(requestObj, function(response) {
if (response.headers["set-cookie"]) {
for (let i = 0 ; i < response.headers["set-cookie"].length ; ++i) {
for (let i = 0; i < response.headers["set-cookie"].length; ++i) {
cookieJar.update(response.headers["set-cookie"][i]);
}
}
if (response.statusCode < 300 || response.statusCode < 400 && !client.redirect || response.statusCode >= 400 && client.ignoreErrors) {
let decoder;
if (response.headers["content-encoding"] === "gzip") {
@@ -249,10 +250,22 @@ module.exports = function Browser (_parameters) {
decoder = response;
}
fulfill(response, decoder);
fulfill({
response: response,
body: decoder,
url: url.toString()
});
}
else if (response.statusCode < 400 && _parameters.redirect) {
instance.get(response.headers.location)
else if (response.statusCode < 400 && client.redirect) {
client.headers["Referer"] = url.protocol + "//" + url.hostname + url.pathname;
let newLocation = response.headers.location;
if (newLocation.startsWith("/")) {
newLocation = url.protocol + "//" + url.hostname + newLocation;
}
client.get(newLocation)
.then(fulfill)
.catch(reject);
}
@@ -261,7 +274,7 @@ module.exports = function Browser (_parameters) {
}
});
request.on("error", function(error) {
request.on("error", (error) => {
reject(error);
});
@@ -269,7 +282,7 @@ module.exports = function Browser (_parameters) {
if (_request.data instanceof stream) {
_request.data.pipe(request);
}
else if (_request.data instanceof String || typeof(_request.data) === "string") {
else if (_request.data instanceof String || typeof _request.data === "string") {
request.write(_request.data);
}
else {