From cff7cc2e9c13ec0057fccc4f03d3d46cbb154b60 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Thu, 20 Feb 2020 19:46:39 +0100 Subject: [PATCH 1/3] apply prettier --- app/crawler/specificCrawlers/rental.js | 8 ++++++-- app/models/searchRequest.js | 10 +--------- app/services/notificationService.js | 4 ++-- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js index 39eb1c5..ae0fcd7 100644 --- a/app/crawler/specificCrawlers/rental.js +++ b/app/crawler/specificCrawlers/rental.js @@ -399,7 +399,9 @@ class RentalCrawler { ); if (!publishedDateMoment.isValid()) { throw { - message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}` + message: `Invalid published date : ${ + extractedData["re_realEstates_inserted"] + }` }; } @@ -410,7 +412,9 @@ class RentalCrawler { ); if (!renewedDateMoment.isValid()) { throw { - message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}` + message: `Invalid renewed date : ${ + extractedData["re_realEstates_edited"] + }` }; } diff --git a/app/models/searchRequest.js b/app/models/searchRequest.js index 0d4997f..8ef9b90 100644 --- a/app/models/searchRequest.js +++ b/app/models/searchRequest.js @@ -15,15 +15,7 @@ module.exports = (sequelize, DataTypes) => { allowNull: false, defaultValue: { type: "Polygon", - coordinates: [ - [ - [0, 0], - [0, 0], - [0, 0], - [0, 0], - [0, 0] - ] - ], + coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]], crs: { type: "name", properties: { name: "EPSG:4326" } } } }, diff --git a/app/services/notificationService.js b/app/services/notificationService.js index a005f84..bb649ea 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -131,7 +131,7 @@ const notifyRequestsWithDailyOption = async () => { }; const checkUpNotify = async () => { - /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp(); + /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp(); const asyncSendEmailActions = []; @@ -144,7 +144,7 @@ const checkUpNotify = async () => { asyncSendEmailActions.push(sendEmailPromise); sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err)); } - await Promise.all(asyncSendEmailActions); */ + await Promise.all(asyncSendEmailActions); */ }; module.exports = { -- 2.47.3 From f62a7200c73097ffe3731a4f2520595f796ff0a2 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Thu, 20 Feb 2020 19:47:30 +0100 Subject: [PATCH 2/3] create fetch wrapper with mandatory user agent header --- app/helpers/fetchWrapper.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 app/helpers/fetchWrapper.js diff --git a/app/helpers/fetchWrapper.js b/app/helpers/fetchWrapper.js new file mode 100644 index 0000000..5a660e2 --- /dev/null +++ b/app/helpers/fetchWrapper.js @@ -0,0 +1,13 @@ +const nodeFetch = require("node-fetch"); + +const fetch = async (url, options = {}) => { + const newOptions = Object.assign({}, options); + if (!newOptions["headers"]) { + newOptions["headers"] = {}; + } + newOptions["headers"]["User-Agent"] = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Safari/537.36"; + return nodeFetch(url, newOptions); +}; + +module.exports = fetch; -- 2.47.3 From 4f230020d746d5ed659e945eded3c80b3fdec6e3 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Thu, 20 Feb 2020 19:49:29 +0100 Subject: [PATCH 3/3] use fetch wrapper instead of node-fetch --- app/crawler/specificCrawlers/aktido.js | 2 +- app/crawler/specificCrawlers/olx.js | 2 +- app/crawler/specificCrawlers/prostor.js | 2 +- app/crawler/specificCrawlers/rental.js | 2 +- app/crawler/specificCrawlers/saljic.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/crawler/specificCrawlers/aktido.js b/app/crawler/specificCrawlers/aktido.js index 74bcba7..9755eb7 100644 --- a/app/crawler/specificCrawlers/aktido.js +++ b/app/crawler/specificCrawlers/aktido.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const Promise = require("bluebird"); const moment = require("moment-timezone"); diff --git a/app/crawler/specificCrawlers/olx.js b/app/crawler/specificCrawlers/olx.js index ede9734..1e3d63a 100644 --- a/app/crawler/specificCrawlers/olx.js +++ b/app/crawler/specificCrawlers/olx.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const Promise = require("bluebird"); const moment = require("moment-timezone"); diff --git a/app/crawler/specificCrawlers/prostor.js b/app/crawler/specificCrawlers/prostor.js index 104d5d0..5690a4b 100644 --- a/app/crawler/specificCrawlers/prostor.js +++ b/app/crawler/specificCrawlers/prostor.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const moment = require("moment-timezone"); const FormData = require("form-data"); diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js index ae0fcd7..ed37e83 100644 --- a/app/crawler/specificCrawlers/rental.js +++ b/app/crawler/specificCrawlers/rental.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const Promise = require("bluebird"); const moment = require("moment-timezone"); diff --git a/app/crawler/specificCrawlers/saljic.js b/app/crawler/specificCrawlers/saljic.js index 2eef7cc..9914f35 100644 --- a/app/crawler/specificCrawlers/saljic.js +++ b/app/crawler/specificCrawlers/saljic.js @@ -1,6 +1,6 @@ "use strict"; -const fetch = require("node-fetch"); +const fetch = require("../../helpers/fetchWrapper"); const cheerio = require("cheerio"); const moment = require("moment-timezone"); -- 2.47.3