From 208faa08df037f4d427ef158bbccd3ede2d5166a Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Tue, 25 Jun 2019 17:06:07 +0200 Subject: [PATCH 1/5] Added send notification service, and queried unsent marketalerts, fixed some issues with crawler, and added proper logging --- app/helpers/crawlers/olxClawler.js | 27 ++++----- app/helpers/db/dbHelper.js | 55 +++++++++++++++++-- app/helpers/url.js | 11 ---- ...-notification-sent-boolean-marketalerts.js | 20 +++++++ app/models/marketalert.js | 1 + app/services/crawlerService.js | 22 ++++++-- app/services/notificationService.js | 19 +++++++ package.json | 3 +- 8 files changed, 123 insertions(+), 35 deletions(-) create mode 100644 app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js create mode 100644 app/services/notificationService.js diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 1f7ea1b..4c0a0a0 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -87,14 +87,14 @@ module.exports = class OlxCrawler { const data = { realEstateType: this.getCategoryId(realEstateType), - email : email, + email: email, olxId: olxId, // category: category, url, title, price: isNaN(parsedPrice) ? 0 : parsedPrice, size: parseFloat(size), - gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize), + gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize), address, region, municipality, @@ -151,22 +151,22 @@ module.exports = class OlxCrawler { } } - getCategoryId (category) { + getCategoryId(category) { - switch(category) { + switch (category) { case 'Stanovi': - return 'stan'; + return 'stan'; case 'Vikendice': - return 'vikendica' + return 'vikendica' case 'Kuće': return 'kuca'; default: - return ''; - } - } + return ''; + } + } async indexPages(urls, start, end, maxResults = 1000) { //TODO fix paging @@ -186,16 +186,18 @@ module.exports = class OlxCrawler { } async crawl() { + console.log("OLX CRAWLER: start crawl"); const filteredResults = []; - const realestateRequests = await allRERequest() + const realestateRequests = await allRERequest(); + console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests"); const urls = this.createRequestUrls(realestateRequests); let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults); for (const result of results) { for (const finalResult of result) { if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") { - const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat]); + const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email); if (pointInsideBoundingBox[0].length !== 0) { filteredResults.push(finalResult); @@ -203,8 +205,7 @@ module.exports = class OlxCrawler { } } } - - console.log(filteredResults); + console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length); return filteredResults; } diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js index f51638b..f81961d 100644 --- a/app/helpers/db/dbHelper.js +++ b/app/helpers/db/dbHelper.js @@ -1,15 +1,62 @@ const db = require('../../models/index'); -// TODO Fetch only subscribed realestate requests +/** + * Find all subscribed RealEstateRequests + */ const allRERequest = async () => { - return await db.RealEstateRequest.findAll(); + return await db.RealEstateRequest.findAll({ + where: { + subscribed: true + } + }); } -const findPointInsideBoundingBox = async (latLng) => { - return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))"); + +/** + * Find all unnotified marketalerts, and order them by email + * + * @param notified bolean + * + * @returns array of MarketAlerts + */ +const allMarketAlerts = async (notified) => { + + let queryObject = { + order: [ + ['email', 'DESC'], + ] + } + + if (notified){ + queryObject.where = { + notified: notified + } + } + return await db.MarketAlert.findAll(queryObject); + + // return await db.MarketAlerts.findAll({ + // where: { + // notified: notified + // }, + // order: [ + // ['email', 'DESC'], + // ] + // }); + } + +/** + * Find all unnotified marketalerts + * @param latLng array + * @param email strig + * + * @returns array of MarketAlerts + */ +const findPointInsideBoundingBox = async (latLng, email) => { + return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`); } module.exports = { allRERequest, + allMarketAlerts, findPointInsideBoundingBox }; diff --git a/app/helpers/url.js b/app/helpers/url.js index 854d2a7..8bf1cb5 100644 --- a/app/helpers/url.js +++ b/app/helpers/url.js @@ -7,17 +7,6 @@ const currentRERequest = async (req) => { const request = await db.RealEstateRequest.findOne({ where: {uniqueId} }); return request; }; -// TODO Fetch only subscribed realestate requests -const allRERequest = async () => { - return await db.RealEstateRequest.findAll(); -} - -const findPointInsideBoundingBox = async (latLng) => { - return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))"); -} - module.exports = { currentRERequest, - allRERequest, - findPointInsideBoundingBox }; diff --git a/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js b/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js new file mode 100644 index 0000000..f70e6f6 --- /dev/null +++ b/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js @@ -0,0 +1,20 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn( + 'MarketAlerts', + 'notified', + { + type: Sequelize.BOOLEAN + } + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.removeColumn( + 'MarketAlerts', + 'notified' + ); + } +}; diff --git a/app/models/marketalert.js b/app/models/marketalert.js index 9f1e092..6f58560 100644 --- a/app/models/marketalert.js +++ b/app/models/marketalert.js @@ -11,6 +11,7 @@ module.exports = (sequelize, DataTypes) => { municipality : DataTypes.STRING, region : DataTypes.STRING, realEstateType : DataTypes.STRING, + notified : DataTypes.BOOLEAN, email: { type: DataTypes.STRING, diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 5eb160b..1b75a24 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -2,6 +2,7 @@ const Promise = require("bluebird"); const OlxCrawler = require("../helpers/crawlers/olxClawler"); const db = require("../models/index"); +const { allMarketAlerts } = require('../helpers/db/dbHelper'); const olxCrawler = new OlxCrawler(1, 2, 3); @@ -10,6 +11,7 @@ const crawlers = [ ]; async function crawlAll() { + console.log("CRAWLER SERVICE: crawlAll"); Promise.map(crawlers, function (crawler) { return crawler.crawl(); @@ -17,7 +19,8 @@ async function crawlAll() { try { - const marketAlertsFromDb = await db.MarketAlert.findAll(); + const marketAlertsFromDb = await allMarketAlerts(); + console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length); const marketAlerts = []; const mergedResults = [].concat.apply([], results); @@ -37,16 +40,23 @@ async function crawlAll() { realEstateType: result.realEstateType }) } + console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length); + try { - console.log(marketAlerts); - const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url)); + + const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url)); + console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length); + await db.MarketAlert.bulkCreate(filteredMarketAlerts); - process.exit() + process.exit(); + } catch (e) { - console.log("Could not bulkCreate marketalers reason: ", e); + console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e); + process.exit(); } } catch (e) { - console.log("Error crawling. Trying next crawler! ", e); + console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e); + process.exit(); } }) }; diff --git a/app/services/notificationService.js b/app/services/notificationService.js new file mode 100644 index 0000000..2e4e6c0 --- /dev/null +++ b/app/services/notificationService.js @@ -0,0 +1,19 @@ + +const Promise = require("bluebird"); +const db = require("../models/index"); +const { allMarketAlerts } = require('../helpers/db/dbHelper'); + + +async function processNotifications() { + + try { + const marketAlerts = await allMarketAlerts(false); + console.log(marketAlerts); + process.exit(); + } catch (e) { + console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e); + } +} + + +processNotifications(); \ No newline at end of file diff --git a/package.json b/package.json index 360b7bd..f7a3e66 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "test": "echo \"Error: no test specified\" && exit 1", "start": "node ./index.js", "start-mon": "nodemon ./index.js", - "scheduler": "node ./app/services/crawlerService.js", + "crawler": "node ./app/services/crawlerService.js", + "notification": "node ./app/services/notificationService.js", "migrate": "cd app && npx sequelize db:migrate", "setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate", "docker-start": "docker start pg_marketalerts", -- 2.47.3 From b3baffe1746f400fc18882cd65ef2a24f911fb62 Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Thu, 27 Jun 2019 17:29:57 +0200 Subject: [PATCH 2/5] Send notification email --- app/helpers/awsEmail.js | 101 ++++++++++++++++++++++++++-- app/services/crawlerService.js | 1 - app/services/notificationService.js | 9 ++- 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index f0f01e6..0519c16 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -1,9 +1,10 @@ -const dotenv = require('dotenv'); -dotenv.config(); +const dotenv = require('dotenv').config(); const { getRealEstateTypeEnum } = require('./enums'); const { getRegionName, getMunicipalityName } = require('./codes'); -const AWS = require('aws-sdk'); +var AWS = require('aws-sdk'); +const TEMPLATE_NAME = "MarketAlertTemplate" + AWS.config.update({ region: process.env.AMAZON_REGION, credentials: @@ -77,13 +78,13 @@ Javimi tim. const getEmaiTextVersion = (realestateRequest) => { const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType); - const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : "" + const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : "" const text = "Zdravo, \n Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima \n Ovo je tražena nekretnina: \n , Tip nekretnine: " + realestateRequest.realEstateType + "\n Područje" + getRegionName(realestateRequest.region) + "\n Mjesto " + getMunicipalityName(realestateRequest.region, realestateRequest.municipality) + "\n Kvadratura nekretnine Od " + realestateRequest.sizeMin + " do " + realestateRequest.sizeMaX + + gardenSize - "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax + + "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax + "\n Ako želis prestati dobijati obavještenja za ovu pretragu klikni" + process.env.APP_URL + "/odjava/" + realestateRequest.uniqueId + "\n Ako želiš promijeniti uslove pretrage klikni " + process.env.APP_URL + "/odpregled/" + realestateRequest.uniqueId + "\n Tvoj,\n Javimi tim" @@ -91,6 +92,94 @@ const getEmaiTextVersion = (realestateRequest) => { return text; } +const sendBulkEmail = async (marketAlerts) => { + + try { + // Create the promise and SES service object + // const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).getTemplate({ TemplateName: TEMPLATE_NAME }).promise(); + // const template = await templatePromise; + // console.log(template); + + destinations = [] + + marketAlerts.forEach(marketAlert => { + destinations.push({ + Destination: { + ToAddresses: [ + marketAlert.email + ] + }, + ReplacementTemplateData: `{ "marketAlertUrl":"${marketAlert.url}", "favoriteanimal":"yak" }` + }) + + }); + + + // Create sendBulkTemplatedEmail params + var params = { + Destinations: /* required */ + // { + // Destination: { /* required */ + // CcAddresses: [ + // 'EMAIL_ADDRESS', + // /* more items */ + // ], + // ToAddresses: [ + // 'EMAIL_ADDRESS', + // 'EMAIL_ADDRESS' + // /* more items */ + // ] + // }, + // ReplacementTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }' + // }, + // ], + destinations, + Source: process.env.SOURCE_EMAIL, /* required */ + Template: TEMPLATE_NAME, /* required */ + DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }', + ReplyToAddresses: [ + process.env.SOURCE_EMAIL, + ] + }; + + + + + // Create the promise and SES service object + const sendPromise =new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise(); + + await sendPromise; + + + } catch (e) { + console.log("Could not send bulk email", e) + } + +} + +const createMarketAlertEmailTemplate = async () => { + const marketAlertTemplate = { + Template: { + TemplateName: "MarketAlertTemplate", + SubjectPart: "Greetings", + TextPart: "Dear ,\r\nYour favorite animal is {{marketAlertUrl}}.", + HtmlPart: "

Hello

Your favorite animal is {{marketAlertUrl}}.

" + } + } + + try { + const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).createTemplate(marketAlertTemplate).promise(); + await templatePromise + + } catch (e) { + console.log("Could not create MarketAlertEmailTemplate", e); + } +} + + + module.exports = { - sendTemplatedEmail + sendTemplatedEmail, + sendBulkEmail, + createMarketAlertEmailTemplate }; diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 1b75a24..756e9bf 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -62,4 +62,3 @@ async function crawlAll() { }; crawlAll(); - diff --git a/app/services/notificationService.js b/app/services/notificationService.js index 2e4e6c0..fb342f4 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -2,18 +2,21 @@ const Promise = require("bluebird"); const db = require("../models/index"); const { allMarketAlerts } = require('../helpers/db/dbHelper'); +const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/awsEmail'); async function processNotifications() { try { const marketAlerts = await allMarketAlerts(false); - console.log(marketAlerts); + // await createMarketAlertEmailTemplate(); + await sendBulkEmail(marketAlerts); + // console.log(marketAlerts); + process.exit(); } catch (e) { console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e); } } - -processNotifications(); \ No newline at end of file +processNotifications(); -- 2.47.3 From 96e9da1fb1c811bf884cb669ee1662c804784f78 Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Fri, 28 Jun 2019 18:06:19 +0200 Subject: [PATCH 3/5] Send templated bulk email, and remember notifed marketalerts --- app/helpers/awsEmail.js | 93 +++++++++++-------- app/helpers/crawlers/olxClawler.js | 2 +- app/helpers/db/dbHelper.js | 20 ++-- ...0190628165512-add-title-to-marketalerts.js | 20 ++++ app/models/marketalert.js | 1 + app/services/crawlerService.js | 6 +- app/services/notificationService.js | 17 +++- 7 files changed, 98 insertions(+), 61 deletions(-) create mode 100644 app/migrations/20190628165512-add-title-to-marketalerts.js diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index 0519c16..f72a896 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -28,11 +28,11 @@ const sendTemplatedEmail = async (email, request) => { Body: { /* required */ Html: { Charset: "UTF-8", - Data: getEmailHTML(request) + Data: getGreetingsEmailHTML(request) }, Text: { Charset: "UTF-8", - Data: getEmaiTextVersion(request) + Data: getGreetingsEmaiTextVersion(request) } }, Subject: { @@ -50,7 +50,7 @@ const sendTemplatedEmail = async (email, request) => { await sendEmailPromise; } -const getEmailHTML = (realestateRequest) => { +const getGreetingsEmailHTML = (realestateRequest) => { const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType); const gardenSize = realEstateType.hasGardenSize ? `
Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2
` : `` @@ -76,7 +76,7 @@ Javimi tim. } -const getEmaiTextVersion = (realestateRequest) => { +const getGreetingsEmaiTextVersion = (realestateRequest) => { const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType); const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : "" @@ -95,45 +95,39 @@ const getEmaiTextVersion = (realestateRequest) => { const sendBulkEmail = async (marketAlerts) => { try { - // Create the promise and SES service object - // const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).getTemplate({ TemplateName: TEMPLATE_NAME }).promise(); - // const template = await templatePromise; - // console.log(template); destinations = [] + groupedEmails = []; marketAlerts.forEach(marketAlert => { + if (!groupedEmails[marketAlert.email]) { + groupedEmails[marketAlert.email] = []; + groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); + } else { + groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); + } + }); + + for (email in groupedEmails) { + + const url = groupedEmails[email]; + let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }` + destinations.push({ Destination: { ToAddresses: [ - marketAlert.email + email ] }, - ReplacementTemplateData: `{ "marketAlertUrl":"${marketAlert.url}", "favoriteanimal":"yak" }` + ReplacementTemplateData: repData }) - }); + } + console.log(destinations); - - // Create sendBulkTemplatedEmail params var params = { - Destinations: /* required */ - // { - // Destination: { /* required */ - // CcAddresses: [ - // 'EMAIL_ADDRESS', - // /* more items */ - // ], - // ToAddresses: [ - // 'EMAIL_ADDRESS', - // 'EMAIL_ADDRESS' - // /* more items */ - // ] - // }, - // ReplacementTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }' - // }, - // ], - destinations, + Destinations: + destinations, Source: process.env.SOURCE_EMAIL, /* required */ Template: TEMPLATE_NAME, /* required */ DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }', @@ -142,12 +136,8 @@ const sendBulkEmail = async (marketAlerts) => { ] }; - - - // Create the promise and SES service object - const sendPromise =new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise(); - + const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise(); await sendPromise; @@ -155,20 +145,45 @@ const sendBulkEmail = async (marketAlerts) => { console.log("Could not send bulk email", e) } + +} + +const toAWSArray = (urlArray) => { + let arrayString = "" + urlArray.forEach(element => { + arrayString = arrayString + `{"url":"${element.url}" , "title":"${element.title}"},` + }); + + return arrayString.slice(0, -1);; +} + +const getNotificationEmailHtml = () => { + return `

Zdravo, + Pronašli smo nekretninu koju ste tražili.

+

Ovo su tražene nekretnine:

+
+
{{#each marketAlertUrl}}
  • {{title}}

    • {{/each}}
    +
    +
    ` +} + +const getNotificationEmailText = () => { + return ` Zdravo, + Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}` } const createMarketAlertEmailTemplate = async () => { const marketAlertTemplate = { Template: { TemplateName: "MarketAlertTemplate", - SubjectPart: "Greetings", + SubjectPart: "Javi mi obavijest", TextPart: "Dear ,\r\nYour favorite animal is {{marketAlertUrl}}.", - HtmlPart: "

    Hello

    Your favorite animal is {{marketAlertUrl}}.

    " + HtmlPart: getNotificationEmailHtml() } } try { - const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).createTemplate(marketAlertTemplate).promise(); + const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise(); await templatePromise } catch (e) { @@ -176,8 +191,6 @@ const createMarketAlertEmailTemplate = async () => { } } - - module.exports = { sendTemplatedEmail, sendBulkEmail, diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 4c0a0a0..556d9f3 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -27,7 +27,7 @@ module.exports = class OlxCrawler { // } //TODO remove properties that are not needed, and add some if they are missing - const title = $('#naslovartikla').text(); + const title = $('#naslovartikla').text().trim(); const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text(); const price = $('#pc > p:nth-child(2)').text(); diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js index f81961d..3363e1b 100644 --- a/app/helpers/db/dbHelper.js +++ b/app/helpers/db/dbHelper.js @@ -11,15 +11,15 @@ const allRERequest = async () => { }); } - /** - * Find all unnotified marketalerts, and order them by email + * Find all , or all depending on notified bolean marketalerts, and order them by email * + * @param fechAll bolean * @param notified bolean * * @returns array of MarketAlerts */ -const allMarketAlerts = async (notified) => { +const allMarketAlerts = async (fetchAll, notified) => { let queryObject = { order: [ @@ -27,27 +27,19 @@ const allMarketAlerts = async (notified) => { ] } - if (notified){ + if (!fetchAll){ queryObject.where = { notified: notified } } - return await db.MarketAlert.findAll(queryObject); - // return await db.MarketAlerts.findAll({ - // where: { - // notified: notified - // }, - // order: [ - // ['email', 'DESC'], - // ] - // }); + return await db.MarketAlert.findAll(queryObject); } /** * Find all unnotified marketalerts * @param latLng array - * @param email strig + * @param email string * * @returns array of MarketAlerts */ diff --git a/app/migrations/20190628165512-add-title-to-marketalerts.js b/app/migrations/20190628165512-add-title-to-marketalerts.js new file mode 100644 index 0000000..d874f3d --- /dev/null +++ b/app/migrations/20190628165512-add-title-to-marketalerts.js @@ -0,0 +1,20 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn( + 'MarketAlerts', + 'title', + { + type: Sequelize.STRING + } + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.removeColumn( + 'MarketAlerts', + 'title' + ); + } +}; diff --git a/app/models/marketalert.js b/app/models/marketalert.js index 6f58560..c23961b 100644 --- a/app/models/marketalert.js +++ b/app/models/marketalert.js @@ -12,6 +12,7 @@ module.exports = (sequelize, DataTypes) => { region : DataTypes.STRING, realEstateType : DataTypes.STRING, notified : DataTypes.BOOLEAN, + title : DataTypes.STRING, email: { type: DataTypes.STRING, diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 756e9bf..57d667a 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -19,7 +19,7 @@ async function crawlAll() { try { - const marketAlertsFromDb = await allMarketAlerts(); + const marketAlertsFromDb = await allMarketAlerts(true); console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length); const marketAlerts = []; @@ -37,7 +37,9 @@ async function crawlAll() { municipality: result.municipality, region: result.region, gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize, - realEstateType: result.realEstateType + realEstateType: result.realEstateType, + title: result.title, + notified: false }) } console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length); diff --git a/app/services/notificationService.js b/app/services/notificationService.js index fb342f4..14d45d4 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -8,11 +8,20 @@ const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/aw async function processNotifications() { try { - const marketAlerts = await allMarketAlerts(false); - // await createMarketAlertEmailTemplate(); - await sendBulkEmail(marketAlerts); - // console.log(marketAlerts); + const marketAlerts = await allMarketAlerts(false, false); + console.log(marketAlerts.length) + await createMarketAlertEmailTemplate(); + if (marketAlerts.length > 0) { + console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length) + await sendBulkEmail(marketAlerts); + } else { + console.log("NOTIFICATION SERVICE: No new alerts"); + } + await db.MarketAlert.update( + { notified: true }, /* set attributes' value */ + { where: { notified: false } } /* where criteria */ + ); process.exit(); } catch (e) { console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e); -- 2.47.3 From 93c147e73b5faafe15c64f9030d6d5d3c2f640cd Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Tue, 2 Jul 2019 11:54:33 +0200 Subject: [PATCH 4/5] Looged amazon send bulk email response, fixed some emails not sent bug --- app/helpers/awsEmail.js | 11 +++++++---- app/services/notificationService.js | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index f72a896..218a368 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -123,6 +123,7 @@ const sendBulkEmail = async (marketAlerts) => { }) } + console.log("AWS EMAIL : Bulk email replacement data:"); console.log(destinations); var params = { @@ -138,7 +139,9 @@ const sendBulkEmail = async (marketAlerts) => { // Create the promise and SES service object const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise(); - await sendPromise; + const awsResult = await sendPromise; + console.log("AWS SES bulk email response"); + console.log(awsResult); } catch (e) { @@ -151,10 +154,10 @@ const sendBulkEmail = async (marketAlerts) => { const toAWSArray = (urlArray) => { let arrayString = "" urlArray.forEach(element => { - arrayString = arrayString + `{"url":"${element.url}" , "title":"${element.title}"},` + arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${element.title.replace(/"/g, "")}"},` }); - return arrayString.slice(0, -1);; + return arrayString.slice(0, -1); } const getNotificationEmailHtml = () => { @@ -177,7 +180,7 @@ const createMarketAlertEmailTemplate = async () => { Template: { TemplateName: "MarketAlertTemplate", SubjectPart: "Javi mi obavijest", - TextPart: "Dear ,\r\nYour favorite animal is {{marketAlertUrl}}.", + TextPart: getNotificationEmailText(), HtmlPart: getNotificationEmailHtml() } } diff --git a/app/services/notificationService.js b/app/services/notificationService.js index 14d45d4..f299330 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -16,6 +16,7 @@ async function processNotifications() { await sendBulkEmail(marketAlerts); } else { console.log("NOTIFICATION SERVICE: No new alerts"); + return; } await db.MarketAlert.update( -- 2.47.3 From 8b20f0e1704ab93cd85e3af9c0e0582c3132b1d1 Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Tue, 2 Jul 2019 12:25:22 +0200 Subject: [PATCH 5/5] Formated title --- app/helpers/awsEmail.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index 218a368..e9ffe9d 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -154,7 +154,8 @@ const sendBulkEmail = async (marketAlerts) => { const toAWSArray = (urlArray) => { let arrayString = "" urlArray.forEach(element => { - arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${element.title.replace(/"/g, "")}"},` + const formatetdTitle = element.title.replace(/"/g, ""); + arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},` }); return arrayString.slice(0, -1); -- 2.47.3