diff --git a/lib/httpclient.js b/lib/httpclient.js index 627096a..2aa293a 100644 --- a/lib/httpclient.js +++ b/lib/httpclient.js @@ -1,10 +1,12 @@ "use strict"; -var http = require("http"); -var https = require("https"); -var url = require("url"); -var querystring = require("querystring"); -var stream = require("stream"); +const http = require("http"); +const https = require("https"); +const { URL } = require('url'); +const querystring = require("querystring"); +const stream = require("stream"); +const zlib = require('zlib'); + function Cookie(rawCookie) { var cookieRegex = /([^,; =]+)(?:=([^;]*)(?:[ ;]+|$))/g; @@ -16,23 +18,23 @@ function Cookie(rawCookie) { var key = match[1]; var value = match[2]; - switch (key) { - case "Path" : + switch (key.toLowerCase()) { + case "path" : cookie.path = value; break; - case "Domain" : + case "domain" : cookie.domain = value; break; - case "Expires" : - this.expires = Date.parse(value); + case "expires" : + cookie.expires = Date.parse(value); break; - case "Max-Age" : + case "max-age" : cookie.maxAge = Number.parseInt(value); break; - case "Secured" : + case "secured" : cookie.secured = true; break; - case "Http-Only" : + case "http-only" : cookie.httpOnly = true; break; default : @@ -55,8 +57,8 @@ var CookieJar = function () { var currentDate = new Date(); cookies.get(path).forEach( function (value, key, map) { - if ((value.expires !== undefined && value.expires < currentDate) - || (value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date())) { + if (value.expires !== undefined && value.expires < currentDate + || value.maxAge !== undefined && currentDate.setSeconds(currentDate.getSeconds() + value.maxAge) < new Date()) { map.delete(key); } else { @@ -77,8 +79,8 @@ var CookieJar = function () { cookies.set(path, new Map()); } - var deleteCookie = ((cookie.maxAge !== undefined && cookie.maxAge <= 0) - || (cookie.expires !== undefined && cookie.expires < new Date())); + var deleteCookie = cookie.maxAge !== undefined && cookie.maxAge <= 0 + || cookie.expires !== undefined && cookie.expires < new Date(); if (deleteCookie) { if (cookies.get(path).has(cookie.name)) { @@ -95,146 +97,145 @@ var CookieJar = function () { }; -module.exports = function Browser (_parameters) { - var baseUrl = null; - var instance = this; +module.exports = function Browser (parameters) { + parameters = parameters || {}; + + var baseUrl = parameters.baseUrl ? new URL(parameters.baseUrl) : null; //CookieJar and custom headers var cookieJar = new CookieJar(); - this.headers = {}; - - if (_baseUrl) { - baseUrl = url.parse(_parameters.baseUrl); - } - - if (_headers) { - this.headers = _parameters.headers; - } - + this.headers = parameters.headers || {}; + this.redirect = parameters && parameters.redirect; + this.ignoreErrors = parameters && parameters.ignoreErrors; + if (this.headers["User-Agent"] === undefined) { - this.headers["User-Agent"] = "HttpClient/1.0.0 (" + os.type() + " " + os.release()+")"; + this.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0"; } if (this.headers["Accept-Language"] === undefined) { - this.headers["Accept-Language"] = "en-US"; + this.headers["Accept-Language"] = "en,*:q=0.1"; } if (this.headers["Accept"] === undefined) { - this.headers["Accept"] = "text/plain,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + this.headers["Accept"] = "*/*"; } if (this.headers["Accept-Encoding"] === undefined) { this.headers["Accept-Encoding"] = "gzip,deflate"; } - this.get = function (_path, _querystring) { + if (this.headers["Accept-Charset"] === undefined) { + this.headers["Accept-Charset"] = "us-ascii,ISO-8859-1,ISO-8859-2,ISO-8859-3,ISO-8859-4,ISO-8859-5,ISO-8859-6,ISO-8859-7,ISO-8859-8,ISO-8859-9,ISO-8859-10,ISO-8859-13,ISO-8859-14,ISO-8859-15,ISO-8859-16,windows-1250,windows-1251,windows-1252,windows-1256,windows-1257,cp437,cp737,cp850,cp852,cp866,x-cp866-u,x-mac,x-mac-ce,x-kam-cs,koi8-r,koi8-u,koi8-ru,TCVN-5712,VISCII,utf-8"; + } + + if (parameters.keepAlive) { + this.headers["Connection"] = "keep-alive"; + } + + this.get = function (path, querystring) { return query({ method : "GET", - path : _path, - querystring : _querystring, - }); + path: path, + querystring: querystring + }, this); }; - this.post = function(_path, _data) { + this.post = function (path, data) { return query({ method : "POST", - path : _path, - data : _data - }); + path: path, + data: data + }, this); }; - this.postForm = function(_path, _data) { + this.postForm = function (path, data) { return query({ method : "POST", - path : _path, - data : querystring.stringify(_data) - }); + path: path, + data: querystring.stringify(data) + }, this); }; - this.put = function(_path, _data) { + this.put = function (path, data) { return query({ method : "PUT", - path : _path, - data : _data - }); + path: path, + data: data + }, this); }; - this.delete = function(_path, _querystring) { + this.delete = function (path, querystring) { return query({ method : "DELETE", - path : _path, - querystring : _querystring - }); + path: path, + querystring: querystring + }, this); }; - this.patch = function(_path, _data) { + this.patch = function (path, data) { return query({ method : "PATCH", - path : _path, - data : _data - }); + path: path, + data: data + }, this); }; - function query(_request) { - var pathUrl = url.parse(_request.path); - - pathUrl.headers = instance.headers; + function query(_request, client) { + var url = new URL(_request.path); + var isHttps = url.protocol === "https:"; + + var requestObj = { + hostname: url.hostname, + port: url.port ? url.port : isHttps ? 443 : 80, + path: url.pathname, + method: _request.method, + headers: client.headers + }; + + if (url.search) { + requestObj.path += url.search; + } + if (_request.querystring) { let encodedQueryString = querystring.stringify(_request.querystring); - pathUrl.search = pathUrl.search ? pathUrl.search + "&" + encodedQueryString : "?" + encodedQueryString; + requestObj.path += url.search ? url.search + "&" + encodedQueryString : "?" + encodedQueryString; } - var cookies = cookieJar.get(pathUrl.pathname); + var cookies = cookieJar.get(url.pathname); - if ( pathUrl.pathname !== "/") { + if (url.pathname !== "/") { cookies += cookieJar.get("/"); } if (cookies) { - pathUrl.headers.Cookies = cookies; + requestObj.headers.Cookies = cookies; } + + if (isHttps) { + requestObj.rejectUnauthorized = false; + } + //Complete the url with the base one if a path was provided - if (baseUrl !== null && pathUrl.protocol === null) { - pathUrl.protocol = baseUrl.protocol; - pathUrl.hostname = baseUrl.hostname; - pathUrl.auth = baseUrl.auth; - pathUrl.port = baseUrl.port; - } + // if (client.baseUrl !== null && pathUrl.protocol === null) { + // pathUrl.protocol = baseUrl.protocol; + // pathUrl.hostname = baseUrl.hostname; + // pathUrl.auth = baseUrl.auth; + // pathUrl.port = baseUrl.port; + // } - return new Promise(function(fulfill, reject) { - var protocol; - - if (pathUrl.protocol === "https:") { - protocol = https; - - if (pathUrl.rejectUnauthorized === null || pathUrl.rejectUnauthorized === undefined) { - pathUrl.rejectUnauthorized = false; - } - - if (!pathUrl.port) { - pathUrl.port = 443; - } - } - else { - protocol = http; - - if (!pathUrl.port) { - pathUrl.port = 80; - } - } - - pathUrl.method = _request.method; + return new Promise((fulfill, reject) => { + var protocol = isHttps ? https : http; - var request = protocol.request(pathUrl, function(response) { - if (response.statusCode < 300 || (response.statusCode < 400 && !_parameters.redirect)) { - if (response.headers["set-cookie"]) { - for (let i = 0 ; i < response.headers["set-cookie"].length ; ++i) { - cookieJar.update(response.headers["set-cookie"][i]); - } + var request = protocol.request(requestObj, function(response) { + if (response.headers["set-cookie"]) { + for (let i = 0; i < response.headers["set-cookie"].length; ++i) { + cookieJar.update(response.headers["set-cookie"][i]); } + } + if (response.statusCode < 300 || response.statusCode < 400 && !client.redirect || response.statusCode >= 400 && client.ignoreErrors) { let decoder; if (response.headers["content-encoding"] === "gzip") { @@ -249,10 +250,22 @@ module.exports = function Browser (_parameters) { decoder = response; } - fulfill(response, decoder); + fulfill({ + response: response, + body: decoder, + url: url.toString() + }); } - else if (response.statusCode < 400 && _parameters.redirect) { - instance.get(response.headers.location) + else if (response.statusCode < 400 && client.redirect) { + client.headers["Referer"] = url.protocol + "//" + url.hostname + url.pathname; + + let newLocation = response.headers.location; + + if (newLocation.startsWith("/")) { + newLocation = url.protocol + "//" + url.hostname + newLocation; + } + + client.get(newLocation) .then(fulfill) .catch(reject); } @@ -261,15 +274,15 @@ module.exports = function Browser (_parameters) { } }); - request.on("error", function(error) { + request.on("error", (error) => { reject(error); }); - + if (_request.data) { if (_request.data instanceof stream) { _request.data.pipe(request); } - else if (_request.data instanceof String || typeof(_request.data) === "string") { + else if (_request.data instanceof String || typeof _request.data === "string") { request.write(_request.data); } else {