senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'add-email-frequency-option' into 'master'

Browse Source 
Add email frequency option

See merge request saburly/marketalarm/web!68
This commit was merged in pull request #68.
This commit is contained in:
Bilal Catic
2019-11-04 13:28:44 +00:00
parent e3e0ddd508 4d5571b1d8
commit 8ea44f5fc7
15 changed files with 251 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/common/enums.js
View File

@@ -174,10 +174,24 @@ const CRAWLER_AD_TYPE = {
ONLY_REQUEST: 4
};
const EMAIL_FREQUENCY = {
ASAP: {
id: 1,
stringId: "ASAP",
title: "Odmah"
},
DAILY: {
id: 2,
stringId: "DAILY",
title: "Jednom dnevno"
}
};
module.exports = {
AD_TYPE,
AD_CATEGORY,
AD_STATUS,
AD_AGENCY,
CRAWLER_AD_TYPE
CRAWLER_AD_TYPE,
EMAIL_FREQUENCY
};

40
app/controllers/queryReview.js
View File

@@ -3,9 +3,9 @@ const { isValidEmail } = require("../helpers/email");
const {
notifyForNewSearchRequest
} = require("../services/notificationService");
const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
const getQueryReviewData = searchRequest => {
const getQueryReviewTableData = searchRequest => {
const {
id,
adType,
@@ -87,15 +87,26 @@ const getQueryReview = async (req, res) => {
const title = "Da li je ovo to što ste tražili ?";
const nextStep = req.query.nextStep;
const error = req.query.error;
const queryReviewData = getQueryReviewData(searchRequest);
const queryReviewTableData = getQueryReviewTableData(searchRequest);
const email = searchRequest.email;
let selectedEmailFrequency;
switch (searchRequest.emailFrequency) {
case EMAIL_FREQUENCY.ASAP.stringId:
selectedEmailFrequency = EMAIL_FREQUENCY.ASAP.id;
break;
case EMAIL_FREQUENCY.DAILY.stringId:
selectedEmailFrequency = EMAIL_FREQUENCY.DAILY.id;
break;
}
res.render("queryReview", {
nextStep,
queryReviewData,
queryReviewTableData,
title,
email,
error
selectedEmailFrequency,
error,
EMAIL_FREQUENCY
});
};
@@ -107,17 +118,26 @@ const postQueryReview = async (req, res) => {
}
const nextStep = req.query.nextStep || "/ponovo";
const emailFrequency =
parseInt(req.body.emailFrequency) || EMAIL_FREQUENCY.ASAP.id;
const emailInput = req.body.email;
const emailConfirmInput = req.body.confirmEmail;
const title = "Da li je ovo to što ste tražili ?";
const queryReviewData = getQueryReviewData(searchRequest);
const queryReviewTableData = getQueryReviewTableData(searchRequest);
let emailFrequencyStringId = EMAIL_FREQUENCY.ASAP.stringId;
if (emailFrequency === EMAIL_FREQUENCY.DAILY.id) {
emailFrequencyStringId = EMAIL_FREQUENCY.DAILY.stringId;
}
searchRequest.emailFrequency = emailFrequencyStringId;
if (emailInput !== emailConfirmInput) {
const error = "Greška ! Unešeni emailovi nisu isti";
res.render("queryReview", {
error,
title,
queryReviewData,
queryReviewTableData,
email: ""
});
return;
@@ -128,7 +148,7 @@ const postQueryReview = async (req, res) => {
res.render("queryReview", {
error,
title,
queryReviewData,
queryReviewTableData,
email: ""
});
return;
@@ -147,7 +167,7 @@ const postQueryReview = async (req, res) => {
res.render("queryReview", {
error,
title,
queryReviewData,
queryReviewTableData,
email: ""
});
return;
@@ -164,7 +184,7 @@ const postQueryReview = async (req, res) => {
res.render("queryReview", {
error,
title,
queryReviewData,
queryReviewTableData,
email: ""
});
return;

20
app/helpers/db/searchRequestMatch.js
View File

@@ -24,6 +24,23 @@ const findRealEstatesForSearchRequest = async searchRequestId => {
return matchingRealEstates;
};
const findNotNotifiedMatches = async () => {
const query = {
notified: false
};
const searchRequestsModel = { model: db.SearchRequest, as: "searchRequests" };
const realEstateModel = { model: db.RealEstate, as: "realEstates" };
const include = [searchRequestsModel, realEstateModel];
const matchingRecords = await db.SearchRequestMatch.findAll({
where: query,
include
});
return matchingRecords;
};
const addMatches = async matchingRecords => {
return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
ignoreDuplicates: true
@@ -32,5 +49,6 @@ const addMatches = async matchingRecords => {
module.exports = {
findRealEstatesForSearchRequest,
addMatches
addMatches,
findNotNotifiedMatches
};

19
app/helpers/emailContentGenerator.js
View File

@@ -20,7 +20,11 @@ const generateRealEstateLinks = realEstates => {
return realEstateLinks;
};
const generateNotificationEmail = (realEstates, searchRequestId) => {
const generateNotificationEmail = (
realEstates,
searchRequestId,
dailyNotification = false
) => {
const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
const realEstatesToShow = truncateList
? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
@@ -30,9 +34,20 @@ const generateNotificationEmail = (realEstates, searchRequestId) => {
const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
const emailFooter = generateEmailFooter(searchRequestId);
const asapMessageBody =
realEstates.length > 1
? "Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi"
: "Pronašli smo nekretninu koja odgovara Vašoj pretrazi";
const dailyMessageBody =
realEstates.length > 1
? "U posljednja 24h objavljene su sljedeće nekretnine koje odgovaraju uslovima Vaše pretrage"
: "U posljednja 24h objavljena je sljedeća nekretnina koja odgovara uslovima Vaše pretrage";
const messageBody = dailyNotification ? dailyMessageBody : asapMessageBody;
return `<h3>Zdravo</h3>
<h4>Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi</h4>
<h4>${messageBody}</h4>
<div>
${realEstateLinks}
<div/>

15
app/migrations/20191101113541-add-emailFrequency-column-to-searchRequest-table.js Normal file
View File

@@ -0,0 +1,15 @@
"use strict";
const { EMAIL_FREQUENCY } = require("../common/enums");
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("SearchRequests", "emailFrequency", {
type: Sequelize.TEXT,
defaultValue: EMAIL_FREQUENCY.ASAP.stringId
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("SearchRequests", "emailFrequency");
}
};

7
app/models/searchRequest.js
View File

@@ -1,6 +1,6 @@
"use strict";
const { AD_TYPE } = require("../common/enums");
const { AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
module.exports = (sequelize, DataTypes) => {
const SearchRequest = sequelize.define("SearchRequest", {
@@ -61,6 +61,11 @@ module.exports = (sequelize, DataTypes) => {
type: DataTypes.BOOLEAN,
defaultValue: false,
allowNull: false
},
emailFrequency: {
type: DataTypes.TEXT,
defaultValue: EMAIL_FREQUENCY.ASAP.stringId,
allowNull: false
}
});

6
app/models/searchRequestMatch.js
View File

@@ -44,6 +44,12 @@ module.exports = (sequelize, DataTypes) => {
);
SearchRequestMatch.associate = models => {
SearchRequestMatch.hasMany(models.SearchRequest, {
foreignKey: "id",
sourceKey: "searchRequestId",
targetKey: "id",
as: "searchRequests"
});
SearchRequestMatch.hasMany(models.RealEstate, {
foreignKey: "id",
as: "realEstates"

8
app/npmScripts/npmDailyNotify.js Normal file
View File

@@ -0,0 +1,8 @@
"use strict";
const {
notifyRequestsWithDailyOption
} = require("../services/notificationService");
(async () => {
await notifyRequestsWithDailyOption();
})();

19
app/public/segment.css
View File

@@ -1,14 +1,18 @@
.ui-segment {
.segmented {
color: #02adba;
border: 1px solid #02adba;
border-radius: 4px;
display: inline-block;
}
.ui-segment span.option.active {
.segmented label {
color: #02adba;
}
.segmented input:checked + .label {
background-color: #02adba;
color: white;
}
.ui-segment span.option {
[type="radio"]:not(:checked) + span,
[type="radio"]:checked + span {
padding-left: 30px;
padding-right: 30px;
height: 35px;
@@ -21,9 +25,14 @@
border-right: 1px solid #02adba;
}
.ui-segment span.option:last-child {
.segmented :last-child .label {
border-right: none;
}
.segment-select {
.segmented input {
display: none;
}
span.label:before,
span.label:after {
display: none;
}

88
app/services/notificationService.js
View File

@@ -8,6 +8,7 @@ const {
generateNewSearchRequestEmail,
generateEmailSubject
} = require("../helpers/emailContentGenerator");
const { findNotNotifiedMatches } = require("../helpers/db/searchRequestMatch");
const { sendEmail } = require("../services/emailService");
const notifyForNewRealEstates = async newRealEstates => {
@@ -29,34 +30,87 @@ const notifyForNewSearchRequest = async searchRequest => {
await sendEmail(email, "Kivi - novi zahtjev za pretragu", emailContent);
};
const notifyMatches = async matches => {
const notifyMatches = async (matches, dailyNotification = false) => {
const searchRequestsToNotify = Object.keys(matches);
const asyncSendEmailActions = [];
for (const id of searchRequestsToNotify) {
const { searchRequest } = matches[id];
const { email } = searchRequest;
const allMatchingRealEstates = matches[id].realEstates || [];
if (allMatchingRealEstates.length > 0) {
const emailContent = generateNotificationEmail(
allMatchingRealEstates,
id
);
const emailSubject = generateEmailSubject(
allMatchingRealEstates.length,
allMatchingRealEstates[0].title
);
const { searchRequest, notifyNow } = matches[id];
if (notifyNow) {
const { email } = searchRequest;
const allMatchingRealEstates = matches[id].realEstates || [];
if (allMatchingRealEstates.length > 0) {
const emailContent = generateNotificationEmail(
allMatchingRealEstates,
id,
dailyNotification
);
const emailSubject = generateEmailSubject(
allMatchingRealEstates.length,
allMatchingRealEstates[0].title
);
const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
asyncSendEmailActions.push(sendEmailPromise);
sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
asyncSendEmailActions.push(sendEmailPromise);
sendEmailPromise.catch(err =>
console.log("[Email Sending Failed]", err)
);
}
}
}
await Promise.all(asyncSendEmailActions);
};
const notifyRequestsWithDailyOption = async () => {
const notNotifiedSearchRequestMatches = await findNotNotifiedMatches();
const matches = {};
for (const searchRequestMatch of notNotifiedSearchRequestMatches) {
const { searchRequests, realEstates } = searchRequestMatch;
if (!Array.isArray(searchRequests) || searchRequests.length !== 1) {
// Something is wrong with this match
// (search request not found for specified search request id)
// OR
// there are multiple search requests with the same ID (this should never be the case !
// TODO: Maybe if association is defined better, this will be automatically only one object instead of array
continue;
}
if (!Array.isArray(realEstates) || realEstates.length !== 1) {
// Something is wrong with this match
// (real estate not found for specified real estate id)
// OR
// there are multiple real estates with the same ID (this should never be the case !
// TODO: Maybe if association is defined better, this will be automatically only one object instead of array
continue;
}
const searchRequest = searchRequests[0];
const realEstate = realEstates[0];
const searchRequestId = searchRequest.id;
if (!matches[searchRequestId]) {
matches[searchRequestId] = {
searchRequest,
realEstates: [],
notifyNow: true
};
}
matches[searchRequestId].realEstates.push(realEstate);
searchRequestMatch.notified = true;
searchRequestMatch.save();
}
await notifyMatches(matches, true);
};
module.exports = {
notifyForNewRealEstates,
notifyForNewSearchRequest
notifyForNewSearchRequest,
notifyRequestsWithDailyOption
};

10
app/services/searchMatchService.js
View File

@@ -6,6 +6,7 @@ const {
const { findRealEstatesForSearchRequest } = require("../helpers/db/realEstate");
const { addMatches } = require("../helpers/db/searchRequestMatch");
const { MAX_REAL_ESTATES_IN_FIRST_EMAIL } = require("../config/appConfig");
const { EMAIL_FREQUENCY } = require("../common/enums");
const matchRealEstates = async realEstates => {
if (Array.isArray(realEstates)) {
@@ -18,18 +19,19 @@ const matchRealEstates = async realEstates => {
searchRequestsPromise.then(searchRequests => {
for (const searchRequest of searchRequests) {
const { id } = searchRequest;
const { id, emailFrequency } = searchRequest;
if (!matches[id]) {
matches[id] = {
searchRequest,
realEstates: []
realEstates: [],
notifyNow: emailFrequency === EMAIL_FREQUENCY.ASAP.stringId
};
}
matches[id].realEstates.push(realEstate);
matchingRecords.push({
searchRequestId: searchRequest.id,
realEstateId: realEstate.id,
notified: false
notified: emailFrequency === EMAIL_FREQUENCY.ASAP.stringId
});
}
});
@@ -62,7 +64,7 @@ const matchSearchRequest = async searchRequest => {
matchingRecords.push({
searchRequestId,
realEstateId: realEstate.id,
notified: false
notified: true
});
}

23
app/views/queryReview.ejs
View File

@@ -2,7 +2,7 @@
<form method="POST" id="form-queryreview">
<div class="row center-align">
<ul class="collection with-header">
<% for(const stepData of queryReviewData) { %>
<% for(const stepData of queryReviewTableData) { %>
<li class="collection-item" >
<div id="<%= stepData.id %>" ><%= stepData.title || '-' %>
<a href="<%= stepData.url %>" class="kivi-color secondary-content">
@@ -13,6 +13,26 @@
<% } %>
</ul>
</div>
<div class="row center-align">
<h6>Slanje obavještenja</h6>
<span class="segmented">
<label>
<input type="radio" name="emailFrequency" value="<%= EMAIL_FREQUENCY.ASAP.id %>"
<% if (selectedEmailFrequency === EMAIL_FREQUENCY.ASAP.id) { %>
checked
<% } %>>
<span class="label"><%= EMAIL_FREQUENCY.ASAP.title %></span>
</label>
<label>
<input type="radio" name="emailFrequency" value="<%= EMAIL_FREQUENCY.DAILY.id %>"
<% if (selectedEmailFrequency === EMAIL_FREQUENCY.DAILY.id) { %>
checked
<% } %>>
<span class="label"><%= EMAIL_FREQUENCY.DAILY.title %></span>
</label>
</span>
</div>
<div class="row center-align">
<div class="col">
<input id="email" name="email" type="email" placeholder="vas.email@mail.com" <% if (email) { %>value="<%= email %>" <% } %> required size="250" />
@@ -47,6 +67,7 @@
</form>
<script>
$(document).ready( () => {
$("#submit").click( () => {
const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;

68
app/views/realEstateType.ejs
View File

@@ -3,18 +3,23 @@
<div class="center-align">
<div class="row">
<select class="segment-select" id="adType" name="adType">
<option value="<%= AD_TYPE.AD_TYPE_SALE.id %>"
<% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
selected="selected"
<% } %>
><%= AD_TYPE.AD_TYPE_SALE.title %></option>
<option value="<%= AD_TYPE.AD_TYPE_RENT.id %>"
<% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
selected="selected"
<% } %>
><%= AD_TYPE.AD_TYPE_RENT.title %></option>
</select>
<span class="segmented">
<label>
<input type="radio" name="adType" value="<%= AD_TYPE.AD_TYPE_SALE.id %>"
<% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
checked
<% } %>>
<span class="label"><%= AD_TYPE.AD_TYPE_SALE.title %></span>
</label>
<label>
<input type="radio" name="adType" value="<%= AD_TYPE.AD_TYPE_RENT.id %>"
<% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
checked
<% } %>>
<span class="label"><%= AD_TYPE.AD_TYPE_RENT.title %></span>
</label>
</span>
</div>
<br>
@@ -37,45 +42,6 @@
</form>
<script>
(function($) {
$.fn.extend({
Segment: function() {
$(this).each(function() {
const self = $(this);
const onchange = self.attr('onchange');
const wrapper = $("<div>", { class: "ui-segment" });
$(this)
.find("option")
.each(function() {
const option = $("<span>", {
class: "option",
onclick: onchange,
text: $(this).text(),
value: $(this).val(),
});
if ($(this).is(":selected")) {
option.addClass("active");
}
wrapper.append(option);
});
wrapper.find("span.option").click(function (){
wrapper.find("span.option").removeClass("active");
$(this).addClass("active");
self.val($(this).attr('value'));
});
$(this).after(wrapper);
$(this).hide();
});
}
});
})(jQuery);
$(document).ready(() => {
$(".segment-select").Segment();
});
function saveAndSubmit(id) {
$("#realEstateType").val(id);
$("#realEstateTypeSelection > a").attr("onclick", "");

4
development.env
View File

@@ -31,6 +31,7 @@ OLX_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check commo
OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
#==RENTAL==
RENTAL_MAX_PAGES=Restrict crawler to this number of pages
RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -38,6 +39,7 @@ RENTAL_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check co
RENTAL_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
RENTAL_IGNORED_USERNAMES=!!! This is not used for rental crawler !!!
RENTAL_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
RENTAL_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
#==PROSTOR==
PROSTOR_MAX_PAGES=!!! This is not used for prostor crawler !!!
PROSTOR_MAX_RESULTS_PER_PAGE=For Prostor crawler, this represents MAX RESULTS in total
@@ -45,6 +47,7 @@ PROSTOR_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check c
PROSTOR_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
PROSTOR_IGNORED_USERNAMES=!!! This is not used for prostor crawler !!!
PROSTOR_DELAY_BETWEEN_PAGES=!!! This is not used for prostor crawler !!!
PROSTOR_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
#==AKTIDO==
AKTIDO_MAX_PAGES=Restrict crawler to this number of pages
AKTIDO_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -52,3 +55,4 @@ AKTIDO_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check co
AKTIDO_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found

1
package.json
View File

@@ -12,6 +12,7 @@
"docker-start": "docker start pg_marketalerts",
"docker-stop": "docker stop pg_marketalerts",
"crawl": "cd app/crawler && node npmCrawl.js",
"daily-notify": "cd app/npmScripts && node npmDailyNotify.js",
"test-search": "cd test && node searchTest.js",
"test-olx-scraper": "cd test && node olxScrapeTest.js"
},