Compare commits
14 Commits
crawler-op
...
email-resu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a457226207 | ||
|
|
4f35305b0a | ||
|
|
09792db21c | ||
|
|
1999d45cb2 | ||
|
|
778b5ff411 | ||
|
|
81c30c36ec | ||
|
|
753a09aa36 | ||
|
|
4517624fa8 | ||
|
|
f9abf48f61 | ||
|
|
afeffe8c71 | ||
|
|
a6bd63b7b8 | ||
|
|
e305c547e1 | ||
|
|
33f9e37d93 | ||
|
|
5829de64e0 |
@@ -3,6 +3,8 @@ const { getRealEstateTypeEnum } = require('../helpers/enums');
|
||||
|
||||
const getGardenSize = (req,res) => {
|
||||
|
||||
const title = "Koliko okućnice tražite ?"
|
||||
|
||||
const unit = " m2"
|
||||
const rangeFrom = {
|
||||
min : 10,
|
||||
@@ -18,7 +20,7 @@ const getGardenSize = (req,res) => {
|
||||
step : 10
|
||||
}
|
||||
|
||||
res.render('gardenSize', { rangeFrom, rangeTo, unit });
|
||||
res.render('gardenSize', { rangeFrom, rangeTo, unit, title });
|
||||
};
|
||||
|
||||
const postGardenSize = async (req, res) => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
const getGoAgain = async (req,res) => {
|
||||
res.render('goAgain');
|
||||
const title = "Želite li pretražiti još jednu nekretninu ?";
|
||||
res.render('goAgain', {title});
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -2,10 +2,12 @@ const { currentRERequest } = require('../helpers/url');
|
||||
const { getMunicipalitiesForRegion, getMunicipalityName } = require('../helpers/codes');
|
||||
|
||||
const getMunicipality = async (req, res) => {
|
||||
|
||||
const title = "U kojem mjestu tražite nekretninu?"
|
||||
let request = await currentRERequest(req);
|
||||
const municipalities = getMunicipalitiesForRegion(request.region);
|
||||
|
||||
res.render('municipality', { municipalities });
|
||||
res.render('municipality', { municipalities, title });
|
||||
};
|
||||
|
||||
const postMunicipality = async (req, res) => {
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
const { currentRERequest } = require('../helpers/url');
|
||||
|
||||
const getNeighborhood = async (req, res) => {
|
||||
|
||||
const title = "U kojem naselju tražite nekretninu?"
|
||||
const municipality = req.params.municipality
|
||||
const nextStep = req.query.nextStep || '/';
|
||||
|
||||
res.render('neighborhoodMap', {
|
||||
nextStep,
|
||||
municipality
|
||||
municipality,
|
||||
title
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
@@ -2,6 +2,8 @@ const { currentRERequest } = require('../helpers/url');
|
||||
|
||||
const getPrice = (req,res) => {
|
||||
|
||||
const title = "Koja Vam okvirna cijena odgovara ?"
|
||||
|
||||
const unit = " KM"
|
||||
const rangeFrom = {
|
||||
min : 1000,
|
||||
@@ -18,7 +20,7 @@ const getPrice = (req,res) => {
|
||||
}
|
||||
|
||||
|
||||
res.render('price', {rangeFrom, rangeTo, unit });
|
||||
res.render('price', {rangeFrom, rangeTo, unit, title });
|
||||
};
|
||||
|
||||
const postPrice = async (req, res) => {
|
||||
|
||||
@@ -3,6 +3,8 @@ const { getRegionName, getMunicipalityName } = require('../helpers/codes');
|
||||
const { realEstateTypes, sizes, gardenSizes, prices, getEnumTypeTitle, getRealEstateTypeEnum } = require('../helpers/enums');
|
||||
|
||||
const getQueryReview = async (req,res) => {
|
||||
|
||||
const title = "Da li je ovo to što ste tražili ?"
|
||||
const request = await currentRERequest(req);
|
||||
const nextStep = req.query.nextStep;
|
||||
|
||||
@@ -69,6 +71,7 @@ const getQueryReview = async (req,res) => {
|
||||
res.render('queryReview', {
|
||||
nextStep,
|
||||
queryData,
|
||||
title
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -3,12 +3,15 @@ const { isValidEmail } = require('../helpers/email');
|
||||
const { sendTemplatedEmail} = require('../helpers/awsEmail');
|
||||
|
||||
const getQuerySubmit = async (req, res) => {
|
||||
|
||||
const title = "Upišite vaš e-mail"
|
||||
const nextStep = req.query.nextStep;
|
||||
const error = req.query.error;
|
||||
|
||||
res.render('querySubmit', {
|
||||
nextStep,
|
||||
error
|
||||
error,
|
||||
title
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -5,7 +5,8 @@ const { realEstateTypes, getRealEstateTypeEnum } = require('../helpers/enums');
|
||||
|
||||
|
||||
const getRealEstateTypes = (req,res) => {
|
||||
res.render('realEstateType', { realEstateTypes });
|
||||
const title = "Koju nekretninu tražite?"
|
||||
res.render('realEstateType', { realEstateTypes, title });
|
||||
};
|
||||
|
||||
const postRealEstateTypes = async (req, res) => {
|
||||
|
||||
18
app/controllers/realEstates.js
Normal file
18
app/controllers/realEstates.js
Normal file
@@ -0,0 +1,18 @@
|
||||
|
||||
const {allMarketAlertsByRequest} = require('../helpers/db/dbHelper');
|
||||
|
||||
const getRealEstates = async (req,res) => {
|
||||
console.log("Enter get realestates");
|
||||
const request = req.params['request_id'];
|
||||
console.log(req.params['request_id']);
|
||||
const realEstates = await allMarketAlertsByRequest(request);
|
||||
console.log(realEstates);
|
||||
|
||||
const title = "Ovo su nekretnine koje smo pronašli za vas"
|
||||
res.render('realEstates', {realEstates, title } );
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getRealEstates
|
||||
};
|
||||
|
||||
@@ -4,7 +4,8 @@ const { getRegions } = require('../helpers/codes');
|
||||
const regions = getRegions();
|
||||
|
||||
const getRegion = (req,res) => {
|
||||
res.render('region', { regions });
|
||||
const title = "U kojoj regiji tražite nekretninu?"
|
||||
res.render('region', { regions, title });
|
||||
};
|
||||
|
||||
const postRegion = async (req, res) => {
|
||||
|
||||
@@ -3,6 +3,7 @@ const { sizes, getRealEstateTypeEnum } = require('../helpers/enums');
|
||||
|
||||
const getSize = (req,res) => {
|
||||
|
||||
const title = "Od koliko kvadrata tražite nekretninu ?"
|
||||
const unit = " m2"
|
||||
const rangeFrom = {
|
||||
min : 10,
|
||||
@@ -18,7 +19,7 @@ const getSize = (req,res) => {
|
||||
step : 10
|
||||
}
|
||||
|
||||
res.render('size', { rangeFrom, rangeTo, unit });
|
||||
res.render('size', { rangeFrom, rangeTo, unit, title });
|
||||
};
|
||||
|
||||
const postSize = async (req, res) => {
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
const { currentRERequest } = require('../helpers/url');
|
||||
|
||||
const getUnsubscribe = async (req, res) => {
|
||||
|
||||
const title = "Uspješno ste se odjavili"
|
||||
const request = await currentRERequest(req);
|
||||
request.subscribed = false;
|
||||
await request.save();
|
||||
|
||||
res.render('unsubscribe', { nextStep: '/vrstanekretnine' });
|
||||
res.render('unsubscribe', { nextStep: '/vrstanekretnine', title });
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
const getWelcome = (req,res) => {
|
||||
res.render('welcome', { nextStep: '/vrstanekretnine' } );
|
||||
const title = ""
|
||||
res.render('welcome', { nextStep: '/vrstanekretnine', title } );
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -4,7 +4,8 @@ const { getRealEstateTypeEnum } = require('./enums');
|
||||
const { getRegionName, getMunicipalityName } = require('./codes');
|
||||
const { allRERequestByUiid } = require('./db/dbHelper');
|
||||
var AWS = require('aws-sdk');
|
||||
const TEMPLATE_NAME = "MarketAlertTemplate"
|
||||
const TEMPLATE_NAME = process.env.AMAZON_NOTIFICATION_EMAIL_TEMPLATE;
|
||||
const NUMBER_OF_RESULTS_TO_SHOW=10;
|
||||
|
||||
AWS.config.update({
|
||||
region: process.env.AMAZON_REGION,
|
||||
@@ -100,25 +101,24 @@ const sendBulkEmail = async (marketAlerts) => {
|
||||
destinations = []
|
||||
groupedRERequests = [];
|
||||
|
||||
|
||||
const RERequestUuidsMaped = marketAlerts.map(marketAlert => marketAlert.request);
|
||||
|
||||
const RERequestUuidsArray = Array.from(new Set(RERequestUuidsMaped));
|
||||
|
||||
const RERequestUuids = RERequestUuidsArray.map(marketAlert => {
|
||||
return { uniqueId: marketAlert }
|
||||
});
|
||||
|
||||
const RERequest = await allRERequestByUiid(RERequestUuids);
|
||||
const RERequests = await allRERequestByUiid(RERequestUuids);
|
||||
const requestDataValues = [];
|
||||
|
||||
RERequest.forEach(RERequest => {
|
||||
RERequests.forEach(RERequest => {
|
||||
var formatedRequest = {};
|
||||
formatedRequest[RERequest.uniqueId] =
|
||||
requestDataValues[RERequest.uniqueId] = {
|
||||
realEstateType: RERequest.realEstateType,
|
||||
region: RERequest.region,
|
||||
municipality: RERequest.municipality
|
||||
municipality: RERequest.municipality,
|
||||
requestUrl : `${process.env.APP_URL}/nekretnine/${RERequest.uniqueId}`
|
||||
};
|
||||
});
|
||||
|
||||
@@ -129,6 +129,8 @@ const sendBulkEmail = async (marketAlerts) => {
|
||||
realEstateType: requestDataValues[marketAlert.request].realEstateType,
|
||||
municipality: requestDataValues[marketAlert.request].municipality,
|
||||
region: requestDataValues[marketAlert.request].region,
|
||||
requestUrl: requestDataValues[marketAlert.request].requestUrl
|
||||
|
||||
}
|
||||
|
||||
if (!groupedRERequests[marketAlert.request]) {
|
||||
@@ -150,8 +152,9 @@ const sendBulkEmail = async (marketAlerts) => {
|
||||
const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title;
|
||||
const region = getRegionName(marketAlert.requestObject.region);
|
||||
const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality);
|
||||
const requestUrl = marketAlert.requestObject.requestUrl
|
||||
|
||||
let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }`
|
||||
let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}", "requestUrl":"${requestUrl}" }`
|
||||
|
||||
destinations.push({
|
||||
Destination: {
|
||||
@@ -192,9 +195,13 @@ const sendBulkEmail = async (marketAlerts) => {
|
||||
const toAWSArray = (urlArray) => {
|
||||
|
||||
let arrayString = ""
|
||||
i = 0
|
||||
urlArray.forEach(element => {
|
||||
const formatetdTitle = element.title.replace(/"/g, "");
|
||||
arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
|
||||
arrayString = i <= NUMBER_OF_RESULTS_TO_SHOW ? arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},` : arrayString;
|
||||
console.log(i);
|
||||
console.log(arrayString);
|
||||
i++
|
||||
});
|
||||
return arrayString.slice(0, -1);
|
||||
|
||||
@@ -207,12 +214,13 @@ const getNotificationEmailHtml = () => {
|
||||
<div>
|
||||
<div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/>
|
||||
<div/>
|
||||
<div>Kompletan spisak nekretnina možete pegledati ovdije: <a href="{{requestUrl}}">Nekretnine</a> <div>
|
||||
</div>`
|
||||
}
|
||||
|
||||
const getNotificationEmailText = () => {
|
||||
return ` Zdravo,
|
||||
Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
|
||||
Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}} , Kompletan spisan nekretnina mozete pegledati ovdije: {{requestUrl}}`
|
||||
}
|
||||
|
||||
const createMarketAlertEmailTemplate = async () => {
|
||||
|
||||
@@ -7,10 +7,8 @@ const Promise = require("bluebird");
|
||||
|
||||
module.exports = class OlxCrawler {
|
||||
//TODO figure best way to handle paging
|
||||
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
|
||||
this.fromPage = fromPage;
|
||||
this.toPage = toPage;
|
||||
this.maxResults = maxResults;
|
||||
constructor(hrefs = []) {
|
||||
this.hrefs = hrefs;
|
||||
}
|
||||
|
||||
async indexPages(urls) {
|
||||
@@ -36,16 +34,23 @@ module.exports = class OlxCrawler {
|
||||
const urls = this.createRequestUrls(realestateRequests);
|
||||
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
|
||||
console.log("Final crawler results");
|
||||
if (results[0]) {
|
||||
console.log(results[0].length);
|
||||
const flatResults = results.flat();
|
||||
console.log(flatResults);
|
||||
if (flatResults) {
|
||||
console.log(flatResults.length);
|
||||
|
||||
for (const finalResult of results[0]) {
|
||||
for (const finalResult of flatResults) {
|
||||
|
||||
if (null !== finalResult) {
|
||||
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
|
||||
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
|
||||
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email, finalResult.uuid);
|
||||
|
||||
|
||||
if (pointInsideBoundingBox[0].length !== 0) {
|
||||
finalResult.hasLocation = true
|
||||
filteredResults.push(finalResult);
|
||||
} else {
|
||||
finalResult.hasLocation = false
|
||||
filteredResults.push(finalResult);
|
||||
}
|
||||
}
|
||||
@@ -73,7 +78,8 @@ module.exports = class OlxCrawler {
|
||||
const olxUrl = {
|
||||
url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`,
|
||||
email: request.email,
|
||||
uuid: request.uniqueId
|
||||
uuid: request.uniqueId,
|
||||
hrefs: this.hrefs
|
||||
}
|
||||
console.log(olxUrl.url);
|
||||
urls.push(olxUrl);
|
||||
@@ -160,6 +166,7 @@ class Indexer {
|
||||
}
|
||||
|
||||
prepareIndexers(pageNr) {
|
||||
|
||||
console.log("Entering prepareIndexers : page nr - " + pageNr);
|
||||
const indexers = [];
|
||||
let lastPageNumber;
|
||||
@@ -169,7 +176,8 @@ class Indexer {
|
||||
const newOlxUrl = {
|
||||
url: this.olxUrl.url.replace(/\d+$/, "") + index,
|
||||
email: this.olxUrl.email,
|
||||
uuid: this.olxUrl.uuid
|
||||
uuid: this.olxUrl.uuid,
|
||||
hrefs: this.olxUrl.hrefs
|
||||
}
|
||||
indexers.push(new Indexer(newOlxUrl));
|
||||
|
||||
@@ -180,7 +188,8 @@ class Indexer {
|
||||
const newOlxUrl = {
|
||||
url: this.olxUrl.url + index,
|
||||
email: this.olxUrl.email,
|
||||
uuid: this.olxUrl.uuid
|
||||
uuid: this.olxUrl.uuid,
|
||||
hrefs: this.olxUrl.hrefs
|
||||
}
|
||||
indexers.push(new Indexer(newOlxUrl));
|
||||
}
|
||||
@@ -199,7 +208,8 @@ class Indexer {
|
||||
const newOlxUrl = {
|
||||
url: href,
|
||||
email: results.olxUrl.email,
|
||||
uuid: results.olxUrl.uuid
|
||||
uuid: results.olxUrl.uuid,
|
||||
hrefs: this.olxUrl.hrefs
|
||||
}
|
||||
|
||||
indexers.push(new Indexer(newOlxUrl));
|
||||
@@ -212,11 +222,11 @@ class Indexer {
|
||||
|
||||
if (result !== null && result.hasOwnProperty('hrefs')) {
|
||||
result.hrefs.forEach(href => {
|
||||
// console.log(href);
|
||||
const newOlxUrl = {
|
||||
url: href,
|
||||
email: result.olxUrl.email,
|
||||
uuid: result.olxUrl.uuid
|
||||
uuid: result.olxUrl.uuid,
|
||||
hrefs: this.olxUrl.hrefs
|
||||
}
|
||||
|
||||
indexers.push(new Indexer(newOlxUrl));
|
||||
@@ -270,6 +280,16 @@ class Indexer {
|
||||
if (this.olxUrl.url === undefined) {
|
||||
return {}
|
||||
}
|
||||
|
||||
// if (global.hrefs) {
|
||||
|
||||
if (this.olxUrl.hrefs[this.olxUrl.uuid] && this.olxUrl.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
|
||||
|
||||
console.log("We found duplicate URL");
|
||||
return null
|
||||
}
|
||||
// }
|
||||
|
||||
const res = await fetch(this.olxUrl.url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
@@ -26,7 +26,7 @@ const allRERequestByUiid = async (requestArray) => {
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all , or all depending on notified bolean marketalerts, and order them by email
|
||||
* Find all , or all depending on notified bolean marketalerts, that the hasLocation is true, and order them by email
|
||||
*
|
||||
* @param fechAll bolean
|
||||
* @param notified bolean
|
||||
@@ -43,12 +43,32 @@ const allMarketAlerts = async (fetchAll, notified) => {
|
||||
|
||||
if (!fetchAll){
|
||||
queryObject.where = {
|
||||
notified: notified
|
||||
notified: notified,
|
||||
hasLocation: true
|
||||
}
|
||||
}
|
||||
|
||||
return await db.MarketAlert.findAll(queryObject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all , MarketAlerts depending on request
|
||||
*
|
||||
* @param request string
|
||||
*
|
||||
* @returns array of MarketAlerts
|
||||
*/
|
||||
const allMarketAlertsByRequest = async (request) => {
|
||||
|
||||
let queryObject = {
|
||||
where : {
|
||||
request: request
|
||||
}
|
||||
}
|
||||
|
||||
return await db.MarketAlert.findAll(queryObject);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Find all unnotified marketalerts
|
||||
@@ -57,13 +77,14 @@ const allMarketAlerts = async (fetchAll, notified) => {
|
||||
*
|
||||
* @returns array of MarketAlerts
|
||||
*/
|
||||
const findPointInsideBoundingBox = async (latLng, email) => {
|
||||
return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
|
||||
const findPointInsideBoundingBox = async (latLng, email, uniqueId) => {
|
||||
return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND "uniqueId" = '${uniqueId}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
allRERequest,
|
||||
allMarketAlerts,
|
||||
allRERequestByUiid,
|
||||
findPointInsideBoundingBox
|
||||
findPointInsideBoundingBox,
|
||||
allMarketAlertsByRequest
|
||||
};
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = {
|
||||
up: (queryInterface, Sequelize) => {
|
||||
return queryInterface.addColumn(
|
||||
'MarketAlerts',
|
||||
'hasLocation',
|
||||
{
|
||||
type: Sequelize.BOOLEAN
|
||||
}
|
||||
);
|
||||
},
|
||||
|
||||
down: (queryInterface, Sequelize) => {
|
||||
return queryInterface.removeColumn(
|
||||
'MarketAlerts',
|
||||
'hasLocation'
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -14,6 +14,7 @@ module.exports = (sequelize, DataTypes) => {
|
||||
notified : DataTypes.BOOLEAN,
|
||||
title : DataTypes.STRING,
|
||||
request: DataTypes.STRING,
|
||||
hasLocation: DataTypes.BOOLEAN,
|
||||
|
||||
email: {
|
||||
type: DataTypes.STRING,
|
||||
|
||||
@@ -2,6 +2,12 @@
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.next-center-button {
|
||||
width: 50%;
|
||||
left: 25%;
|
||||
}
|
||||
|
||||
|
||||
.welcome-big-logo {
|
||||
font-size: 200pt;
|
||||
background-image: url(./images/logo.png);
|
||||
@@ -13,6 +19,14 @@
|
||||
width: 95%
|
||||
}
|
||||
|
||||
.centered-element {
|
||||
margin-top: 200px;
|
||||
}
|
||||
|
||||
.centered-element-small {
|
||||
margin-top: 100px;
|
||||
}
|
||||
|
||||
|
||||
#map {
|
||||
height: 50%;
|
||||
@@ -21,8 +35,7 @@
|
||||
html,
|
||||
body {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
|
||||
}
|
||||
|
||||
#floating-panel {
|
||||
|
||||
@@ -4,76 +4,82 @@ const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
||||
const db = require("../models/index");
|
||||
const { allMarketAlerts } = require('../helpers/db/dbHelper');
|
||||
|
||||
const olxCrawler = new OlxCrawler(1, 2, 3);
|
||||
|
||||
const crawlers = [
|
||||
olxCrawler,
|
||||
];
|
||||
|
||||
async function crawlAll() {
|
||||
console.log("CRAWLER SERVICE: crawlAll");
|
||||
|
||||
return Promise.map(crawlers, function (crawler) {
|
||||
return crawler.crawl();
|
||||
}).then(async (results) => {
|
||||
try {
|
||||
const marketAlertsFromDb = await allMarketAlerts(true);
|
||||
const hrefs = [];
|
||||
|
||||
try {
|
||||
|
||||
const marketAlertsFromDb = await allMarketAlerts(true);
|
||||
const hrefs = [];
|
||||
const subscribedMakretAlerts = marketAlertsFromDb.filter(marketAlert => {
|
||||
return marketAlert.subscribed;
|
||||
});
|
||||
marketAlertsFromDb.map(marketAlert => {
|
||||
if (hrefs[marketAlert.request] === undefined) {
|
||||
hrefs[marketAlert.request] = []
|
||||
}
|
||||
|
||||
hrefs[marketAlert.request].push(marketAlert.url);
|
||||
})
|
||||
|
||||
global.hrefs = hrefs;
|
||||
console.log(global.hrefs);
|
||||
|
||||
console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + subscribedMakretAlerts.length);
|
||||
|
||||
const marketAlerts = [];
|
||||
const mergedResults = [].concat.apply([], results);
|
||||
|
||||
for (const result of mergedResults) {
|
||||
marketAlerts.push({
|
||||
url: result.url,
|
||||
realestateOrigin: "OLX",
|
||||
originId: 1,
|
||||
size: result.size,
|
||||
price: result.price,
|
||||
email: result.email,
|
||||
request: result.uuid,
|
||||
// lastDate: DataTypes.STRING,
|
||||
municipality: result.municipality,
|
||||
region: result.region,
|
||||
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
|
||||
realEstateType: result.realEstateType,
|
||||
title: result.title,
|
||||
notified: false
|
||||
})
|
||||
marketAlertsFromDb.map(marketAlert => {
|
||||
if (hrefs[marketAlert.request] === undefined) {
|
||||
hrefs[marketAlert.request] = []
|
||||
}
|
||||
console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
|
||||
|
||||
hrefs[marketAlert.request].push(marketAlert.url);
|
||||
})
|
||||
|
||||
console.log("CRAWLER SERVICE: GLOBAL HREFS");
|
||||
console.log(hrefs);
|
||||
const olxCrawler = new OlxCrawler(hrefs);
|
||||
|
||||
const crawlers = [
|
||||
olxCrawler,
|
||||
];
|
||||
|
||||
return Promise.map(crawlers, function (crawler) {
|
||||
return crawler.crawl();
|
||||
}).then(async (results) => {
|
||||
|
||||
try {
|
||||
|
||||
const filteredMarketAlerts = marketAlerts.filter((elem) => !subscribedMakretAlerts.find(({ url }) => elem.url === url));
|
||||
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
|
||||
const marketAlertsFromDb = await allMarketAlerts(false, true);
|
||||
|
||||
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
|
||||
console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
|
||||
|
||||
const marketAlerts = [];
|
||||
const mergedResults = [].concat.apply([], results);
|
||||
|
||||
for (const result of mergedResults) {
|
||||
marketAlerts.push({
|
||||
url: result.url,
|
||||
realestateOrigin: "OLX",
|
||||
originId: 1,
|
||||
size: result.size,
|
||||
price: result.price,
|
||||
email: result.email,
|
||||
request: result.uuid,
|
||||
municipality: result.municipality,
|
||||
region: result.region,
|
||||
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
|
||||
realEstateType: result.realEstateType,
|
||||
title: result.title,
|
||||
notified: false,
|
||||
hasLocation: result.hasLocation
|
||||
})
|
||||
}
|
||||
console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
|
||||
|
||||
try {
|
||||
|
||||
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => {
|
||||
|
||||
return (elem.url === url && elem.request === request)
|
||||
}));
|
||||
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
|
||||
|
||||
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
|
||||
|
||||
} catch (e) {
|
||||
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
|
||||
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
|
||||
}
|
||||
})
|
||||
})
|
||||
} catch (e) {
|
||||
console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
|
||||
}
|
||||
};
|
||||
module.exports = crawlAll;
|
||||
// crawlAll();
|
||||
|
||||
@@ -1,6 +1,2 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h2>Koliko okućnice tražite ?</h2>
|
||||
</div>
|
||||
|
||||
<% include partials/range %>
|
||||
|
||||
@@ -1,14 +1,6 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h4>Provjerite Vaš email !</h4>
|
||||
</div>
|
||||
|
||||
<div class="row center-align">
|
||||
<h4>Želite li pretražiti još jednu nekretninu ?</h4>
|
||||
</div>
|
||||
|
||||
<form method="POST" id="form-goagain">
|
||||
<div class="row">
|
||||
<div class="row centered-element">
|
||||
<div class="col s3 push-s3">
|
||||
<a href="/" class="welcome-center-button waves-effect waves-light btn">
|
||||
Da
|
||||
|
||||
@@ -1,22 +1,25 @@
|
||||
<!doctype>
|
||||
<!doctype>
|
||||
<html>
|
||||
<head>
|
||||
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.css">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="stylesheet" href="/assets/main.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<%-body%>
|
||||
</div>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/js/materialize.min.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/wnumb/1.1.0/wNumb.min.js"></script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
<head>
|
||||
<link href="https://fonts.googleapis.com/css?family=Roboto&display=swap" rel="stylesheet">
|
||||
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.css">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="stylesheet" href="/assets/main.css">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<% include partials/navBar %>
|
||||
<div class="container">
|
||||
|
||||
<%-body%>
|
||||
</div>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/js/materialize.min.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/wnumb/1.1.0/wNumb.min.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,8 +1,4 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h2>U kojem mjestu tražite nekretninu?</h2>
|
||||
</div>
|
||||
|
||||
<form method="POST" id="form-municipality">
|
||||
<div class="row center-align">
|
||||
<ul class="collection with-header">
|
||||
|
||||
@@ -1,7 +1,3 @@
|
||||
<div class="row center-align">
|
||||
<h2>U kojem naselju tražite nekretninu?</h2>
|
||||
</div>
|
||||
|
||||
<div class="row center-align">
|
||||
<div id="floating-panel">
|
||||
<input id="address" type="textbox" value="">
|
||||
|
||||
7
app/views/partials/navBar.ejs
Normal file
7
app/views/partials/navBar.ejs
Normal file
@@ -0,0 +1,7 @@
|
||||
<div class="row center-align">
|
||||
<nav style="background-color: #26a69a; margin: auto;">
|
||||
<div class="row center-align">
|
||||
<h6 style="padding-top: 20px;"><%= title %></h5>
|
||||
</div>
|
||||
</nav>
|
||||
</div>
|
||||
@@ -1,9 +1,9 @@
|
||||
<form method="POST" id="form-range">
|
||||
|
||||
<div class="row center-align no-ui-slider" id="slider"></div>
|
||||
<div class="row center-align no-ui-slider centered-element-small" id="slider"></div>
|
||||
|
||||
<div class="col s6 push-s3">
|
||||
<a id="btnsubmit" href="#" class="welcome-center-button waves-effect waves-light btn">
|
||||
<div class="col s6 push-s3 centered-element-small">
|
||||
<a id="btnsubmit" href="#" class="next-center-button waves-effect waves-light btn">
|
||||
Dalje
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -1,6 +1,2 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h2>Koja Vam okvirna cijena odgovara ?</h2>
|
||||
</div>
|
||||
|
||||
<% include partials/range %>
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h2>Da li je ovo to što ste tražili ?</h2>
|
||||
</div>
|
||||
|
||||
<form method="POST" id="form-queryreview">
|
||||
<div class="row center-align">
|
||||
<ul class="collection with-header">
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h4>Da Vam javimo kada se Vaša željena nekretnina pojavi u oglasima, upišite svoj e-mail</h4>
|
||||
<h6>Da Vam javimo kada se Vaša željena nekretnina pojavi u oglasima, upišite vaš e-mail</h6>
|
||||
</div>
|
||||
|
||||
<form method="POST" id="form-submitquery">
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h2>Koju nekretninu tražite?</h2>
|
||||
</div>
|
||||
|
||||
<form method="POST" id="form-real-estate-type">
|
||||
<div class="row center-align">
|
||||
<ul class="collection with-header">
|
||||
|
||||
14
app/views/realEstates.ejs
Normal file
14
app/views/realEstates.ejs
Normal file
@@ -0,0 +1,14 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<ul class="collection with-header">
|
||||
<% for(const realEstate of realEstates) { %>
|
||||
<li class="collection-item">
|
||||
<div><%= realEstate.title %>
|
||||
<a href="<%= realEstate.url %>" class="secondary-content">
|
||||
<i class="material-icons">send</i>
|
||||
</a>
|
||||
</div>
|
||||
</li>
|
||||
<% } %>
|
||||
</ul>
|
||||
</div>
|
||||
@@ -1,8 +1,4 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h2>U kojoj regiji tražite nekretninu?</h2>
|
||||
</div>
|
||||
|
||||
<form method="POST" id="form-region">
|
||||
<div class="row center-align">
|
||||
<ul class="collection with-header">
|
||||
|
||||
@@ -1,6 +1,2 @@
|
||||
<!--suppress HtmlUnknownAnchorTarget -->
|
||||
<div class="row center-align">
|
||||
<h2>Od koliko kvadrata tražite nekretninu ?</h2>
|
||||
</div>
|
||||
|
||||
<% include partials/range %>
|
||||
|
||||
@@ -2,9 +2,6 @@
|
||||
<div class="row center-align">
|
||||
<span class="welcome-big-logo">🤙</span>
|
||||
</div>
|
||||
<div class="row center-align">
|
||||
<div>Uspješno ste se odjavili</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col s6 push-s3">
|
||||
<a href="<%= nextStep %>" class="welcome-center-button waves-effect waves-light btn">
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
AMAZON_ACCES_KEY_ID=(your-key-here)
|
||||
AMAZON_SECRET_ACCESS_KEY=(your-key-here)
|
||||
AMAZON_REGION=eu-west-1
|
||||
AMAZON_NOTIFICATION_EMAIL_TEMPLATE=MarketAlertTemplateDevelopment
|
||||
APP_URL=http://localhost:3001
|
||||
SOURCE_EMAIL=info@saburly.com
|
||||
43
index.js
43
index.js
@@ -10,6 +10,7 @@ const { getQuerySubmit, postQuerySubmit } = require('./app/controllers/querySubm
|
||||
const { getGoAgain } = require('./app/controllers/goAgain');
|
||||
const { getNeighborhood, postNeighborhood } = require('./app/controllers/neighborhoodMap');
|
||||
const { getUnsubscribe } = require('./app/controllers/unsubscribe');
|
||||
const { getRealEstates} = require('./app/controllers/realEstates');
|
||||
const schedule = require('node-schedule');
|
||||
const crawlAll = require('./app/services/crawlerService')
|
||||
const processNotifications = require('./app/services/notificationService')
|
||||
@@ -119,25 +120,6 @@ app.post("/api/payforalert", (req, res) => {
|
||||
});
|
||||
});
|
||||
|
||||
var runServices = async () => {
|
||||
|
||||
|
||||
}
|
||||
|
||||
runServices();
|
||||
|
||||
var rule = new schedule.RecurrenceRule();
|
||||
rule.seccond = 1;
|
||||
schedule.scheduleJob(rule, async function () {
|
||||
console.log(new Date(), 'Crawler service started');
|
||||
await crawlAll();
|
||||
console.log(new Date(), 'Crawler service finished, starting Notification service');
|
||||
await processNotifications();
|
||||
console.log(new Date(), 'Notification service finished');
|
||||
});
|
||||
|
||||
|
||||
|
||||
app.get('/', welcome);
|
||||
app.get('/vrstanekretnine/:request_id', getRealEstateTypes);
|
||||
app.get('/vrstanekretnine', getRealEstateTypes);
|
||||
@@ -173,6 +155,29 @@ app.get('/odjava/:request_id', getUnsubscribe);
|
||||
|
||||
app.get('/ponovo', getGoAgain);
|
||||
|
||||
app.get('/nekretnine/:request_id', getRealEstates);
|
||||
|
||||
app.use('/assets', express.static('./app/public'));
|
||||
|
||||
app.listen(port, () => console.log(`Example app listening on port ${port}!`));
|
||||
|
||||
var rule = new schedule.RecurrenceRule();
|
||||
rule.seccond = 1;
|
||||
schedule.scheduleJob(rule, async function () {
|
||||
console.log(new Date(), 'Crawler service started');
|
||||
await crawlAll();
|
||||
console.log(new Date(), 'Crawler service finished, starting Notification service');
|
||||
await processNotifications();
|
||||
console.log(new Date(), 'Notification service finished');
|
||||
});
|
||||
|
||||
/**
|
||||
* Add flat method to Array
|
||||
*/
|
||||
Object.defineProperty(Array.prototype, 'flat', {
|
||||
value: function(depth = 1) {
|
||||
return this.reduce(function (flat, toFlatten) {
|
||||
return flat.concat((Array.isArray(toFlatten) && (depth>1)) ? toFlatten.flat(depth-1) : toFlatten);
|
||||
}, []);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user