From 81c30c36ec5f648ef5c8eb28a05f3e9d375b8944 Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Fri, 12 Jul 2019 18:00:02 +0200 Subject: [PATCH] Added realestate link to bulk email --- app/controllers/realEstates.js | 12 +++- app/helpers/awsEmail.js | 13 ++-- app/helpers/crawlers/olxClawler.js | 38 +++++----- app/helpers/db/dbHelper.js | 22 +++++- app/services/crawlerService.js | 107 ++++++++++++++--------------- app/views/realEstates.ejs | 2 +- index.js | 3 + 7 files changed, 116 insertions(+), 81 deletions(-) diff --git a/app/controllers/realEstates.js b/app/controllers/realEstates.js index 6d3783d..ddb5e40 100644 --- a/app/controllers/realEstates.js +++ b/app/controllers/realEstates.js @@ -1,7 +1,15 @@ -const getRealEstates = (req,res) => { +const {allMarketAlertsByRequest} = require('../helpers/db/dbHelper'); + +const getRealEstates = async (req,res) => { + console.log("Enter get realestates"); + const request = req.params['request_id']; + console.log(req.params['request_id']); + const realEstates = await allMarketAlertsByRequest(request); + console.log(realEstates); + const title = "Ovo su nekretnine koje smo pronašli za vas" - res.render('realEstates', { nextStep: '/nekretnine', title } ); + res.render('realEstates', {realEstates, title } ); }; module.exports = { diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index b7ad3ed..f6ce8c6 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -4,7 +4,7 @@ const { getRealEstateTypeEnum } = require('./enums'); const { getRegionName, getMunicipalityName } = require('./codes'); const { allRERequestByUiid } = require('./db/dbHelper'); var AWS = require('aws-sdk'); -const TEMPLATE_NAME = "MarketAlertTemplate" +const TEMPLATE_NAME = "MarketAlertTemplateDevelopment" AWS.config.update({ region: process.env.AMAZON_REGION, @@ -118,7 +118,8 @@ const sendBulkEmail = async (marketAlerts) => { requestDataValues[RERequest.uniqueId] = { realEstateType: RERequest.realEstateType, region: RERequest.region, - municipality: RERequest.municipality + municipality: RERequest.municipality, + requestUrl : `${process.env.APP_URL}/nekretnine/${RERequest.uniqueId}` }; }); @@ -129,6 +130,8 @@ const sendBulkEmail = async (marketAlerts) => { realEstateType: requestDataValues[marketAlert.request].realEstateType, municipality: requestDataValues[marketAlert.request].municipality, region: requestDataValues[marketAlert.request].region, + requestUrl: requestDataValues[marketAlert.request].requestUrl + } if (!groupedRERequests[marketAlert.request]) { @@ -150,8 +153,9 @@ const sendBulkEmail = async (marketAlerts) => { const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title; const region = getRegionName(marketAlert.requestObject.region); const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality); + const requestUrl = marketAlert.requestObject.requestUrl - let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }` + let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}", "requestUrl":"${requestUrl}" }` destinations.push({ Destination: { @@ -207,12 +211,13 @@ const getNotificationEmailHtml = () => {
{{#each marketAlertUrl}}
  • {{title}}

    • {{/each}}
    +
    Kompletan spisan nekretnina mozete pegledati ovdije: Nekretnine
    ` } const getNotificationEmailText = () => { return ` Zdravo, - Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}` + Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}} , Kompletan spisan nekretnina mozete pegledati ovdije: {{requestUrl}}` } const createMarketAlertEmailTemplate = async () => { diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 42cb57f..dcc90d7 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -7,10 +7,8 @@ const Promise = require("bluebird"); module.exports = class OlxCrawler { //TODO figure best way to handle paging - constructor(fromPage = 0, toPage = 10, maxResults = 1000) { - this.fromPage = fromPage; - this.toPage = toPage; - this.maxResults = maxResults; + constructor(hrefs = []) { + this.hrefs = hrefs; } async indexPages(urls) { @@ -36,8 +34,7 @@ module.exports = class OlxCrawler { const urls = this.createRequestUrls(realestateRequests); let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults); console.log("Final crawler results"); - // console.log(results); - const flatResults = results.flat(); + const flatResults = results.flat(); console.log(flatResults); if (flatResults) { console.log(flatResults.length); @@ -48,7 +45,7 @@ module.exports = class OlxCrawler { if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") { const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email); - + if (pointInsideBoundingBox[0].length !== 0) { finalResult.hasLocation = true filteredResults.push(finalResult); @@ -81,7 +78,8 @@ module.exports = class OlxCrawler { const olxUrl = { url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`, email: request.email, - uuid: request.uniqueId + uuid: request.uniqueId, + hrefs: this.hrefs } console.log(olxUrl.url); urls.push(olxUrl); @@ -168,6 +166,7 @@ class Indexer { } prepareIndexers(pageNr) { + console.log("Entering prepareIndexers : page nr - " + pageNr); const indexers = []; let lastPageNumber; @@ -177,7 +176,8 @@ class Indexer { const newOlxUrl = { url: this.olxUrl.url.replace(/\d+$/, "") + index, email: this.olxUrl.email, - uuid: this.olxUrl.uuid + uuid: this.olxUrl.uuid, + hrefs: this.olxUrl.hrefs } indexers.push(new Indexer(newOlxUrl)); @@ -188,7 +188,8 @@ class Indexer { const newOlxUrl = { url: this.olxUrl.url + index, email: this.olxUrl.email, - uuid: this.olxUrl.uuid + uuid: this.olxUrl.uuid, + hrefs: this.olxUrl.hrefs } indexers.push(new Indexer(newOlxUrl)); } @@ -207,7 +208,8 @@ class Indexer { const newOlxUrl = { url: href, email: results.olxUrl.email, - uuid: results.olxUrl.uuid + uuid: results.olxUrl.uuid, + hrefs: this.olxUrl.hrefs } indexers.push(new Indexer(newOlxUrl)); @@ -220,11 +222,11 @@ class Indexer { if (result !== null && result.hasOwnProperty('hrefs')) { result.hrefs.forEach(href => { - // console.log(href); const newOlxUrl = { url: href, email: result.olxUrl.email, - uuid: result.olxUrl.uuid + uuid: result.olxUrl.uuid, + hrefs: this.olxUrl.hrefs } indexers.push(new Indexer(newOlxUrl)); @@ -279,14 +281,14 @@ class Indexer { return {} } - if (global.hrefs) { + // if (global.hrefs) { - if (global.hrefs[this.olxUrl.uuid] && global.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) { + if (this.olxUrl.hrefs[this.olxUrl.uuid] && this.olxUrl.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) { - console.log("We found duplicate URL"); - return null - } + console.log("We found duplicate URL"); + return null } + // } const res = await fetch(this.olxUrl.url); const body = await res.text(); diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js index d964d54..de696e9 100644 --- a/app/helpers/db/dbHelper.js +++ b/app/helpers/db/dbHelper.js @@ -50,6 +50,25 @@ const allMarketAlerts = async (fetchAll, notified) => { return await db.MarketAlert.findAll(queryObject); } + + /** + * Find all , MarketAlerts depending on request + * + * @param request string + * + * @returns array of MarketAlerts + */ +const allMarketAlertsByRequest = async (request) => { + + let queryObject = { + where : { + request: request + } + } + + return await db.MarketAlert.findAll(queryObject); +} + /** * Find all unnotified marketalerts @@ -66,5 +85,6 @@ module.exports = { allRERequest, allMarketAlerts, allRERequestByUiid, - findPointInsideBoundingBox + findPointInsideBoundingBox, + allMarketAlertsByRequest }; diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 0c9e072..860afa2 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -4,11 +4,6 @@ const OlxCrawler = require("../helpers/crawlers/olxClawler"); const db = require("../models/index"); const { allMarketAlerts } = require('../helpers/db/dbHelper'); -const olxCrawler = new OlxCrawler(1, 2, 3); - -const crawlers = [ - olxCrawler, -]; async function crawlAll() { console.log("CRAWLER SERVICE: crawlAll"); @@ -16,7 +11,7 @@ async function crawlAll() { try { const marketAlertsFromDb = await allMarketAlerts(true); const hrefs = []; - + marketAlertsFromDb.map(marketAlert => { if (hrefs[marketAlert.request] === undefined) { hrefs[marketAlert.request] = [] @@ -25,64 +20,66 @@ async function crawlAll() { hrefs[marketAlert.request].push(marketAlert.url); }) - global.hrefs = hrefs; console.log("CRAWLER SERVICE: GLOBAL HREFS"); - console.log(global.hrefs); + console.log(hrefs); + const olxCrawler = new OlxCrawler(hrefs); - } catch (e) { - console.error("CRAWLER SERVICE:could not fetch marketalerts ", e); - } - - return Promise.map(crawlers, function (crawler) { - return crawler.crawl(); - }).then(async (results) => { - - try { - - const marketAlertsFromDb = await allMarketAlerts(false, true); - - console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length); - - const marketAlerts = []; - const mergedResults = [].concat.apply([], results); - - for (const result of mergedResults) { - marketAlerts.push({ - url: result.url, - realestateOrigin: "OLX", - originId: 1, - size: result.size, - price: result.price, - email: result.email, - request: result.uuid, - // lastDate: DataTypes.STRING, - municipality: result.municipality, - region: result.region, - gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize, - realEstateType: result.realEstateType, - title: result.title, - notified: false, - hasLocation: result.hasLocation - }) - } - console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length); + const crawlers = [ + olxCrawler, + ]; + + return Promise.map(crawlers, function (crawler) { + return crawler.crawl(); + }).then(async (results) => { try { - const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => { + const marketAlertsFromDb = await allMarketAlerts(false, true); - return (elem.url === url && elem.request === request) })); - console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length); + console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length); - await db.MarketAlert.bulkCreate(filteredMarketAlerts); + const marketAlerts = []; + const mergedResults = [].concat.apply([], results); + for (const result of mergedResults) { + marketAlerts.push({ + url: result.url, + realestateOrigin: "OLX", + originId: 1, + size: result.size, + price: result.price, + email: result.email, + request: result.uuid, + municipality: result.municipality, + region: result.region, + gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize, + realEstateType: result.realEstateType, + title: result.title, + notified: false, + hasLocation: result.hasLocation + }) + } + console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length); + + try { + + const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => { + + return (elem.url === url && elem.request === request) + })); + console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length); + + await db.MarketAlert.bulkCreate(filteredMarketAlerts); + + } catch (e) { + console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e); + } } catch (e) { - console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e); + console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e); } - } catch (e) { - console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e); - } - }) + }) + } catch (e) { + console.error("CRAWLER SERVICE:could not fetch marketalerts ", e); + } }; module.exports = crawlAll; -// crawlAll(); diff --git a/app/views/realEstates.ejs b/app/views/realEstates.ejs index 4e1a98e..9dc5fb6 100644 --- a/app/views/realEstates.ejs +++ b/app/views/realEstates.ejs @@ -6,7 +6,7 @@ <% for(const realEstate of realEstates) { %>
  • <%= realEstate.title %> - + send
    diff --git a/index.js b/index.js index c1a0b7e..a15ee99 100644 --- a/index.js +++ b/index.js @@ -10,6 +10,7 @@ const { getQuerySubmit, postQuerySubmit } = require('./app/controllers/querySubm const { getGoAgain } = require('./app/controllers/goAgain'); const { getNeighborhood, postNeighborhood } = require('./app/controllers/neighborhoodMap'); const { getUnsubscribe } = require('./app/controllers/unsubscribe'); +const { getRealEstates} = require('./app/controllers/realEstates'); const schedule = require('node-schedule'); const crawlAll = require('./app/services/crawlerService') const processNotifications = require('./app/services/notificationService') @@ -154,6 +155,8 @@ app.get('/odjava/:request_id', getUnsubscribe); app.get('/ponovo', getGoAgain); +app.get('/nekretnine/:request_id', getRealEstates); + app.use('/assets', express.static('./app/public')); app.listen(port, () => console.log(`Example app listening on port ${port}!`));