diff --git a/README.md b/README.md index 6f2d0f4..2488e3c 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,10 @@ this will create and run postgres image and then execute migrations 5. Run app `npm start` or `npm run start-mon` to run app with automatic restart on code change + + +### AWS SES + + - AWS SES credentials are handled with env vratiables + - Notification emails are sent in batches of 50, by using SES templates + - Make sure that you are using different templates for different envirorments diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index e9ffe9d..ae590c0 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -2,8 +2,10 @@ const dotenv = require('dotenv').config(); const { getRealEstateTypeEnum } = require('./enums'); const { getRegionName, getMunicipalityName } = require('./codes'); +const db = require('../models/index'); +const { allRERequestByUiid } = require('./db/dbHelper'); var AWS = require('aws-sdk'); -const TEMPLATE_NAME = "MarketAlertTemplate" +const TEMPLATE_NAME = "MarketAlertTemplateDevelop" AWS.config.update({ region: process.env.AMAZON_REGION, @@ -37,7 +39,7 @@ const sendTemplatedEmail = async (email, request) => { }, Subject: { Charset: 'UTF-8', - Data: `Javimi Potvrda: ${request.realEstateType} ${getRegionName(request.region)}, ${getMunicipalityName(request.region, request.municipality)}` + Data: `Javimi Potvrda: ${getSubject(request.realEstateType, request.region, request.municipality)}` } }, Source: process.env.SOURCE_EMAIL, /* required */ @@ -97,26 +99,65 @@ const sendBulkEmail = async (marketAlerts) => { try { destinations = [] - groupedEmails = []; + groupedRERequests = []; - marketAlerts.forEach(marketAlert => { - if (!groupedEmails[marketAlert.email]) { - groupedEmails[marketAlert.email] = []; - groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); - } else { - groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); - } + + let RERequestUuids = marketAlerts.map(marketAlert => marketAlert.request); + + RERequestUuids = Array.from(new Set(RERequestUuids)); + + RERequestUuids = RERequestUuids.map(marketAlert => { + return { uniqueId: marketAlert } }); - for (email in groupedEmails) { + const RERequest = await allRERequestByUiid(RERequestUuids); + const requestDataValues = []; - const url = groupedEmails[email]; - let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }` + RERequest.forEach(RERequest => { + var formatedRequest = {}; + formatedRequest[RERequest.uniqueId] = + requestDataValues[RERequest.uniqueId] = { + realEstateType: RERequest.realEstateType, + region: RERequest.region, + municipality: RERequest.municipality + }; + }); + + marketAlerts.forEach(marketAlert => { + + const requestObject = { + email: marketAlert.email, + realEstateType: requestDataValues[marketAlert.request].realEstateType, + municipality: requestDataValues[marketAlert.request].municipality, + region: requestDataValues[marketAlert.request].region, + } + + if (!groupedRERequests[marketAlert.request]) { + groupedRERequests[marketAlert.request] = { + requestObject: requestObject, + marketAlertArray: [] + }; + } + + groupedRERequests[marketAlert.request].marketAlertArray.push({ + url: marketAlert.url, + title: marketAlert.title, + }); + }); + for (request in groupedRERequests) { + + const marketAlert = groupedRERequests[request]; + let extractedData = toAWSArray(marketAlert.marketAlertArray); + const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title; + const region = getRegionName(marketAlert.requestObject.region); + const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality); + + let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }` destinations.push({ Destination: { ToAddresses: [ - email + marketAlert.requestObject.email ] }, ReplacementTemplateData: repData @@ -147,18 +188,17 @@ const sendBulkEmail = async (marketAlerts) => { } catch (e) { console.log("Could not send bulk email", e) } - - } const toAWSArray = (urlArray) => { + let arrayString = "" urlArray.forEach(element => { const formatetdTitle = element.title.replace(/"/g, ""); arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},` }); - return arrayString.slice(0, -1); + } const getNotificationEmailHtml = () => { @@ -179,8 +219,8 @@ const getNotificationEmailText = () => { const createMarketAlertEmailTemplate = async () => { const marketAlertTemplate = { Template: { - TemplateName: "MarketAlertTemplate", - SubjectPart: "Javi mi obavijest", + TemplateName: TEMPLATE_NAME, + SubjectPart: "Javi mi obavijest: {{realestateType}}, {{region}}, {{municipality}}", TextPart: getNotificationEmailText(), HtmlPart: getNotificationEmailHtml() } @@ -195,6 +235,10 @@ const createMarketAlertEmailTemplate = async () => { } } +const getSubject = (realEstateType, region, municipality) => { + return `${getRealEstateTypeEnum(realEstateType).title} ${getRegionName(region)}, ${getMunicipalityName(region, municipality)}` +} + module.exports = { sendTemplatedEmail, sendBulkEmail, diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 556d9f3..18d25bb 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -13,7 +13,7 @@ module.exports = class OlxCrawler { this.maxResults = maxResults; } - async indexSingle(url, email) { + async indexSingle(url, email, uuid) { try { const res = await fetch(url); const body = await res.text(); @@ -81,6 +81,7 @@ module.exports = class OlxCrawler { const parsedPrice = parsePrice(price); + console.log(location); const locationArray = location.split(","); const region = locationArray[0]; const municipality = locationArray[1]; @@ -88,6 +89,7 @@ module.exports = class OlxCrawler { const data = { realEstateType: this.getCategoryId(realEstateType), email: email, + uuid: uuid, olxId: olxId, // category: category, url, @@ -138,7 +140,7 @@ module.exports = class OlxCrawler { for (let i = 0; i < hrefs.length; i++) { console.log(`indexing: ${hrefs[i]}`); - const singleData = await this.indexSingle(hrefs[i], olxUrl.email); + const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid); if (singleData) { results.push(singleData); @@ -223,7 +225,8 @@ module.exports = class OlxCrawler { const olxUrl = { url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`, - email: request.email + email: request.email, + uuid: request.uniqueId } console.log(olxUrl.url); urls.push(olxUrl); diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js index 3363e1b..234aff5 100644 --- a/app/helpers/db/dbHelper.js +++ b/app/helpers/db/dbHelper.js @@ -11,6 +11,20 @@ const allRERequest = async () => { }); } +/** + * Find all subscribed RealEstateRequests by UUID + */ +const allRERequestByUiid = async (requestArray) => { + + const Op = db.Sequelize.Op; + return await db.RealEstateRequest.findAll({ + where: { + subscribed: true, + [Op.or]: requestArray + } + }); +} + /** * Find all , or all depending on notified bolean marketalerts, and order them by email * @@ -50,5 +64,6 @@ const findPointInsideBoundingBox = async (latLng, email) => { module.exports = { allRERequest, allMarketAlerts, + allRERequestByUiid, findPointInsideBoundingBox }; diff --git a/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js b/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js new file mode 100644 index 0000000..b904e6f --- /dev/null +++ b/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js @@ -0,0 +1,20 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn( + 'MarketAlerts', + 'request', + { + type: Sequelize.STRING + } + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.removeColumn( + 'MarketAlerts', + 'request' + ); + } +}; diff --git a/app/models/marketalert.js b/app/models/marketalert.js index c23961b..72c797c 100644 --- a/app/models/marketalert.js +++ b/app/models/marketalert.js @@ -13,6 +13,7 @@ module.exports = (sequelize, DataTypes) => { realEstateType : DataTypes.STRING, notified : DataTypes.BOOLEAN, title : DataTypes.STRING, + request: DataTypes.STRING, email: { type: DataTypes.STRING, diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 57d667a..e7ac968 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -33,6 +33,7 @@ async function crawlAll() { size: result.size, price: result.price, email: result.email, + request: result.uuid, // lastDate: DataTypes.STRING, municipality: result.municipality, region: result.region, diff --git a/app/services/notificationService.js b/app/services/notificationService.js index f299330..00d8fbf 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -16,7 +16,7 @@ async function processNotifications() { await sendBulkEmail(marketAlerts); } else { console.log("NOTIFICATION SERVICE: No new alerts"); - return; + process.exit(); } await db.MarketAlert.update( @@ -26,6 +26,7 @@ async function processNotifications() { process.exit(); } catch (e) { console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e); + process.exit(); } }