From b2c102bc1ad7eb6ec5d5f856234ee9871493949f Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Thu, 23 Jan 2020 15:48:48 +0100 Subject: [PATCH 1/3] Added check up email that everything works. --- app/helpers/db/searchRequestMatch.js | 13 ++++++++- app/helpers/emailContentGenerator.js | 41 +++++++++++++++++++++++++++- app/npmScripts/npmCheckUpNotify.js | 6 ++++ app/services/notificationService.js | 28 +++++++++++++++++-- package.json | 1 + 5 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 app/npmScripts/npmCheckUpNotify.js diff --git a/app/helpers/db/searchRequestMatch.js b/app/helpers/db/searchRequestMatch.js index 11d5cde..9170863 100644 --- a/app/helpers/db/searchRequestMatch.js +++ b/app/helpers/db/searchRequestMatch.js @@ -40,6 +40,16 @@ const findNotNotifiedMatches = async () => { return matchingRecords; }; +const findAllRequestsForCheckUp = async () => { + const query = { + subscribed: true + }; + const allRequestsForCheckUp = await db.SearchRequest.findAll({ + where: query + }); + + return allRequestsForCheckUp; +}; const addMatches = async matchingRecords => { return await db.SearchRequestMatch.bulkCreate(matchingRecords, { @@ -50,5 +60,6 @@ const addMatches = async matchingRecords => { module.exports = { findRealEstatesForSearchRequest, addMatches, - findNotNotifiedMatches + findNotNotifiedMatches, + findAllRequestsForCheckUp }; diff --git a/app/helpers/emailContentGenerator.js b/app/helpers/emailContentGenerator.js index 861bfb2..ee30ee6 100644 --- a/app/helpers/emailContentGenerator.js +++ b/app/helpers/emailContentGenerator.js @@ -120,8 +120,47 @@ const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => { return `Kivi: ${numberOfRealEstates} novih nekretnina`; }; +const generateCheckUpEmail = searchRequest => { + const realEstateType = AD_CATEGORY[searchRequest.realEstateType]; + const { + id, + gardenSizeMin, + gardenSizeMax, + sizeMin, + sizeMax, + priceMin, + priceMax + } = searchRequest; + + const gardenSize = realEstateType.hasGardenSize + ? `
Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2
` + : ``; + + const allRealEstatesLink = `${APP_URL}/nekretnine/${id}`; + const moreRealEstates = `
Kompletan spisak do sada pronađenih nekretnina možete pogledati na listi nekretnina
`; + + const emailFooter = generateEmailFooter(id); + + return `

Zdravo

+
Kivi tim traži nekretnine za Vas i kada to ne vidite.
+
+
Vaša trenutno aktivna pretraga je:
+
+
+
Tip nekretnine: ${realEstateType.title}
+
Kvadratura nekretnine: Od ${sizeMin} do ${sizeMax} m2
+ ${gardenSize} +
Cijena: ${priceMin} do ${priceMax} KM
+
+
+ ${moreRealEstates} +
+ ${emailFooter}`; +}; + module.exports = { generateNotificationEmail, generateNewSearchRequestEmail, - generateEmailSubject + generateEmailSubject, + generateCheckUpEmail }; diff --git a/app/npmScripts/npmCheckUpNotify.js b/app/npmScripts/npmCheckUpNotify.js new file mode 100644 index 0000000..1becb3f --- /dev/null +++ b/app/npmScripts/npmCheckUpNotify.js @@ -0,0 +1,6 @@ +"use strict"; +const { checkUpNotifyAll } = require("../services/notificationService"); + +(async () => { + await checkUpNotifyAll(); +})(); diff --git a/app/services/notificationService.js b/app/services/notificationService.js index 7495e17..10a74df 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -6,9 +6,13 @@ const { const { generateNotificationEmail, generateNewSearchRequestEmail, - generateEmailSubject + generateEmailSubject, + generateCheckUpEmail } = require("../helpers/emailContentGenerator"); -const { findNotNotifiedMatches } = require("../helpers/db/searchRequestMatch"); +const { + findNotNotifiedMatches, + findAllRequestsForCheckUp +} = require("../helpers/db/searchRequestMatch"); const { sendEmail } = require("../services/emailService"); const notifyForNewRealEstates = async newRealEstates => { @@ -109,8 +113,26 @@ const notifyRequestsWithDailyOption = async () => { await notifyMatches(matches, true); }; +const checkUpNotifyAll = async () => { + const searchRequestsForCheckUp = await findAllRequestsForCheckUp(); + + const asyncSendEmailActions = []; + + for (const searchRequest of searchRequestsForCheckUp) { + const { email } = searchRequest.dataValues; + const emailSubject = `Kivi: Mi tražimo nekretnine za vas!`; + const emailContent = generateCheckUpEmail(searchRequest.dataValues); + + const sendEmailPromise = sendEmail(email, emailSubject, emailContent); + asyncSendEmailActions.push(sendEmailPromise); + sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err)); + } + await Promise.all(asyncSendEmailActions); +}; + module.exports = { notifyForNewRealEstates, notifyForNewSearchRequest, - notifyRequestsWithDailyOption + notifyRequestsWithDailyOption, + checkUpNotifyAll }; diff --git a/package.json b/package.json index 75a7cc4..b4ce4a8 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "docker-stop": "docker stop pg_marketalerts", "crawl": "cd app/crawler && node npmCrawl.js", "daily-notify": "cd app/npmScripts && node npmDailyNotify.js", + "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js", "test-search": "cd test && node searchTest.js", "test-olx-scraper": "cd test && node olxScrapeTest.js", "test-rental-scraper": "cd test && node rentalScrapeTest.js" From 40509d2836f346bed74a1678f922eb8d55b9c354 Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Fri, 24 Jan 2020 01:01:10 +0100 Subject: [PATCH 2/3] Changed logic for checkup.For testing. --- app/config/appConfig.js | 5 ++++- app/helpers/db/searchRequestMatch.js | 22 +++++++++++++++++++++- app/npmScripts/npmCheckUpNotify.js | 6 +++--- app/services/notificationService.js | 4 ++-- development.env | 1 + index.js | 7 ++++++- 6 files changed, 37 insertions(+), 8 deletions(-) diff --git a/app/config/appConfig.js b/app/config/appConfig.js index 7a7887b..d269e6f 100644 --- a/app/config/appConfig.js +++ b/app/config/appConfig.js @@ -14,6 +14,8 @@ const DEFAULT_TIMEZONE = "Europe/Sarajevo"; const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60; const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER); +const CHECK_UP_DAYS = parseInt(process.env.CHECK_UP_DAYS) || 10; + const AWS_EMAIL_CONFIG = { REGION: process.env.AWS_REGION || "", CREDENTIALS: { @@ -42,5 +44,6 @@ module.exports = { MAX_REAL_ESTATES_IN_EMAIL, MAX_REAL_ESTATES_IN_FIRST_EMAIL, PRINT_CRAWLER_DEBUG, - API_MAP_KEY + API_MAP_KEY, + CHECK_UP_DAYS }; diff --git a/app/helpers/db/searchRequestMatch.js b/app/helpers/db/searchRequestMatch.js index 9170863..6dfa986 100644 --- a/app/helpers/db/searchRequestMatch.js +++ b/app/helpers/db/searchRequestMatch.js @@ -1,5 +1,6 @@ "use strict"; const db = require("../../models/index"); +const { CHECK_UP_DAYS } = require("../../config/appConfig"); const findRealEstatesForSearchRequest = async searchRequestId => { const query = { @@ -41,8 +42,27 @@ const findNotNotifiedMatches = async () => { return matchingRecords; }; const findAllRequestsForCheckUp = async () => { + //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE + //The ones that received notification for real estate CHECK_UP_DAYS days from now + const date = new Date(); + const checkUpDate = date.getDate() - CHECK_UP_DAYS; + date.setDate(checkUpDate); + const dateQuery = { + createdAt: { + [Op.gte]: date + } + }; + + const excludedRequests = await db.SearchRequestMatch.findAll({ + attributes: ["searchRequestId"], + where: dateQuery + }); + const query = { - subscribed: true + subscribed: true, + id: { + [Op.notIn]: excludedRequests + } }; const allRequestsForCheckUp = await db.SearchRequest.findAll({ where: query diff --git a/app/npmScripts/npmCheckUpNotify.js b/app/npmScripts/npmCheckUpNotify.js index 1becb3f..b69cbf6 100644 --- a/app/npmScripts/npmCheckUpNotify.js +++ b/app/npmScripts/npmCheckUpNotify.js @@ -1,6 +1,6 @@ "use strict"; -const { checkUpNotifyAll } = require("../services/notificationService"); - +const { checkUpNotify } = require("../services/notificationService"); +//For testing pursposes (async () => { - await checkUpNotifyAll(); + await checkUpNotify(); })(); diff --git a/app/services/notificationService.js b/app/services/notificationService.js index 10a74df..d0f382e 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -113,7 +113,7 @@ const notifyRequestsWithDailyOption = async () => { await notifyMatches(matches, true); }; -const checkUpNotifyAll = async () => { +const checkUpNotify = async () => { const searchRequestsForCheckUp = await findAllRequestsForCheckUp(); const asyncSendEmailActions = []; @@ -134,5 +134,5 @@ module.exports = { notifyForNewRealEstates, notifyForNewSearchRequest, notifyRequestsWithDailyOption, - checkUpNotifyAll + checkUpNotify }; diff --git a/development.env b/development.env index 89f0a1e..f8f3d52 100644 --- a/development.env +++ b/development.env @@ -11,6 +11,7 @@ APP_BASE_URL=base url for the app MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email +CHECK_UP_DAYS=Check up email is sent after this number of days without notification #=============== GOOGLE ANALYTICS =============# GA_ID=Google Analytics ID diff --git a/index.js b/index.js index 074099b..abcd14b 100644 --- a/index.js +++ b/index.js @@ -8,10 +8,12 @@ const forceSSL = require("./app/helpers/forceSSL"); const { APP_PORT, CRAWLER_INTERVAL, - STOP_CRAWLER + STOP_CRAWLER, + CHECK_UP_DAYS } = require("./app/config/appConfig"); const routes = require("./app/routes"); const { crawlAll } = require("./app/crawler/crawl"); +const { checkUpNotify } = require("./app/services/notificationService"); const { notifyForNewRealEstates } = require("./app/services/notificationService"); @@ -45,4 +47,7 @@ const crawl = () => { }); } }; + setInterval(crawl, CRAWLER_INTERVAL * 1000); + +setInterval(checkUpNotify, CHECK_UP_DAYS * 24 * 60 * 60 * 1000); From 470f53d29b85a81c1c38f6877a75e2c1ee6602da Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Fri, 24 Jan 2020 16:04:28 +0100 Subject: [PATCH 3/3] Changed to send email only to some users. --- app/helpers/db/searchRequestMatch.js | 13 +++++++++++-- app/helpers/emailContentGenerator.js | 5 ----- index.js | 5 ++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/app/helpers/db/searchRequestMatch.js b/app/helpers/db/searchRequestMatch.js index 6dfa986..794711b 100644 --- a/app/helpers/db/searchRequestMatch.js +++ b/app/helpers/db/searchRequestMatch.js @@ -1,5 +1,7 @@ "use strict"; const db = require("../../models/index"); +const sequelize = require("sequelize"); +const Op = sequelize.Op; const { CHECK_UP_DAYS } = require("../../config/appConfig"); const findRealEstatesForSearchRequest = async searchRequestId => { @@ -53,11 +55,18 @@ const findAllRequestsForCheckUp = async () => { } }; - const excludedRequests = await db.SearchRequestMatch.findAll({ + const excludedMatches = await db.SearchRequestMatch.findAll({ attributes: ["searchRequestId"], - where: dateQuery + where: dateQuery, + order: [["searchRequestId", "ASC"]] }); + const excludedRequestsAll = excludedMatches.map(match => { + return match.dataValues.searchRequestId; + }); + //Removing duplicate search request id-s for optimization + const excludedRequests = [...new Set(excludedRequestsAll)]; + const query = { subscribed: true, id: { diff --git a/app/helpers/emailContentGenerator.js b/app/helpers/emailContentGenerator.js index ee30ee6..4bd2c59 100644 --- a/app/helpers/emailContentGenerator.js +++ b/app/helpers/emailContentGenerator.js @@ -136,9 +136,6 @@ const generateCheckUpEmail = searchRequest => { ? `
Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2
` : ``; - const allRealEstatesLink = `${APP_URL}/nekretnine/${id}`; - const moreRealEstates = `
Kompletan spisak do sada pronađenih nekretnina možete pogledati na listi nekretnina
`; - const emailFooter = generateEmailFooter(id); return `

Zdravo

@@ -153,8 +150,6 @@ const generateCheckUpEmail = searchRequest => {
Cijena: ${priceMin} do ${priceMax} KM

- ${moreRealEstates} -
${emailFooter}`; }; diff --git a/index.js b/index.js index abcd14b..0669e7b 100644 --- a/index.js +++ b/index.js @@ -8,8 +8,7 @@ const forceSSL = require("./app/helpers/forceSSL"); const { APP_PORT, CRAWLER_INTERVAL, - STOP_CRAWLER, - CHECK_UP_DAYS + STOP_CRAWLER } = require("./app/config/appConfig"); const routes = require("./app/routes"); const { crawlAll } = require("./app/crawler/crawl"); @@ -50,4 +49,4 @@ const crawl = () => { setInterval(crawl, CRAWLER_INTERVAL * 1000); -setInterval(checkUpNotify, CHECK_UP_DAYS * 24 * 60 * 60 * 1000); +setInterval(checkUpNotify, 1000 * 60 * 60 * 24);