senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

make user agent header configurable through env variable

Browse Source
This commit is contained in:
Bilal Catic
2020-02-20 21:07:16 +01:00
parent edc6e2bbf7
commit 6791a509d0
3 changed files with 10 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/config/appConfig.js
View File

@@ -41,6 +41,10 @@ const PROSTOR_LOGIN = {
PASSWORD: process.env.PROSTOR_LOGIN_PASS PASSWORD: process.env.PROSTOR_LOGIN_PASS
}; };
const USER_AGENT =
process.env.USER_AGENT ||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
module.exports = { module.exports = {
APP_PORT, APP_PORT,
APP_URL, APP_URL,
@@ -54,5 +58,6 @@ module.exports = {
API_MAP_KEY, API_MAP_KEY,
STAGING, STAGING,
CHECK_UP_DAYS, CHECK_UP_DAYS,
PROSTOR_LOGIN PROSTOR_LOGIN,
USER_AGENT
}; };

4
app/helpers/fetchWrapper.js
View File

@@ -1,12 +1,12 @@
const nodeFetch = require("node-fetch"); const nodeFetch = require("node-fetch");
const { USER_AGENT } = require("../config/appConfig");
const fetch = async (url, options = {}) => { const fetch = async (url, options = {}) => {
const newOptions = Object.assign({}, options); const newOptions = Object.assign({}, options);
if (!newOptions["headers"]) { if (!newOptions["headers"]) {
newOptions["headers"] = {}; newOptions["headers"] = {};
} }
newOptions["headers"]["User-Agent"] = newOptions["headers"]["User-Agent"] = USER_AGENT;
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Safari/537.36";
return nodeFetch(url, newOptions); return nodeFetch(url, newOptions);
}; };

3
development.env
View File

@@ -10,6 +10,7 @@ APP_BASE_URL=base url for the app
ENVIRONMENT=Variable to denote development, staging and production ENVIRONMENT=Variable to denote development, staging and production
USER_AGENT=User agent header to send in fetch requests
MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
@@ -69,4 +70,4 @@ AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without
SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found