diff --git a/app/crawler/specificCrawlers/aktido.js b/app/crawler/specificCrawlers/aktido.js index 74bcba7..9755eb7 100644 --- a/app/crawler/specificCrawlers/aktido.js +++ b/app/crawler/specificCrawlers/aktido.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const Promise = require("bluebird"); const moment = require("moment-timezone"); diff --git a/app/crawler/specificCrawlers/olx.js b/app/crawler/specificCrawlers/olx.js index ede9734..1e3d63a 100644 --- a/app/crawler/specificCrawlers/olx.js +++ b/app/crawler/specificCrawlers/olx.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const Promise = require("bluebird"); const moment = require("moment-timezone"); diff --git a/app/crawler/specificCrawlers/prostor.js b/app/crawler/specificCrawlers/prostor.js index 104d5d0..5690a4b 100644 --- a/app/crawler/specificCrawlers/prostor.js +++ b/app/crawler/specificCrawlers/prostor.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const moment = require("moment-timezone"); const FormData = require("form-data"); diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js index 39eb1c5..ed37e83 100644 --- a/app/crawler/specificCrawlers/rental.js +++ b/app/crawler/specificCrawlers/rental.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const Promise = require("bluebird"); const moment = require("moment-timezone"); @@ -399,7 +399,9 @@ class RentalCrawler { ); if (!publishedDateMoment.isValid()) { throw { - message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}` + message: `Invalid published date : ${ + extractedData["re_realEstates_inserted"] + }` }; } @@ -410,7 +412,9 @@ class RentalCrawler { ); if (!renewedDateMoment.isValid()) { throw { - message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}` + message: `Invalid renewed date : ${ + extractedData["re_realEstates_edited"] + }` }; } diff --git a/app/crawler/specificCrawlers/saljic.js b/app/crawler/specificCrawlers/saljic.js index 2eef7cc..9914f35 100644 --- a/app/crawler/specificCrawlers/saljic.js +++ b/app/crawler/specificCrawlers/saljic.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const moment = require("moment-timezone"); diff --git a/app/helpers/fetchWrapper.js b/app/helpers/fetchWrapper.js new file mode 100644 index 0000000..5a660e2 --- /dev/null +++ b/app/helpers/fetchWrapper.js @@ -0,0 +1,13 @@ +const nodeFetch = require("node-fetch"); + +const fetch = async (url, options = {}) => { + const newOptions = Object.assign({}, options); + if (!newOptions["headers"]) { + newOptions["headers"] = {}; + } + newOptions["headers"]["User-Agent"] = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Safari/537.36"; + return nodeFetch(url, newOptions); +}; + +module.exports = fetch; diff --git a/app/models/searchRequest.js b/app/models/searchRequest.js index 0d4997f..8ef9b90 100644 --- a/app/models/searchRequest.js +++ b/app/models/searchRequest.js @@ -15,15 +15,7 @@ module.exports = (sequelize, DataTypes) => { allowNull: false, defaultValue: { type: "Polygon", - coordinates: [ - [ - [0, 0], - [0, 0], - [0, 0], - [0, 0], - [0, 0] - ] - ], + coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]], crs: { type: "name", properties: { name: "EPSG:4326" } } } }, diff --git a/app/services/notificationService.js b/app/services/notificationService.js index a005f84..bb649ea 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -131,7 +131,7 @@ const notifyRequestsWithDailyOption = async () => { }; const checkUpNotify = async () => { - /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp(); + /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp(); const asyncSendEmailActions = []; @@ -144,7 +144,7 @@ const checkUpNotify = async () => { asyncSendEmailActions.push(sendEmailPromise); sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err)); } - await Promise.all(asyncSendEmailActions); */ + await Promise.all(asyncSendEmailActions); */ }; module.exports = {