Added realestate link to bulk email

app/controllers/realEstates.js

@@ -1,7 +1,15 @@
 
-const getRealEstates = (req,res) => {
+const {allMarketAlertsByRequest} = require('../helpers/db/dbHelper');
+
+const getRealEstates = async (req,res) => {
+    console.log("Enter get realestates");
+    const request = req.params['request_id'];
+    console.log(req.params['request_id']);
+    const realEstates = await allMarketAlertsByRequest(request);
+    console.log(realEstates);
+
     const title = "Ovo su nekretnine koje smo pronašli za vas"
-    res.render('realEstates', { nextStep: '/nekretnine', title } );
+    res.render('realEstates', {realEstates, title } );
   };
   
   module.exports = {

app/helpers/awsEmail.js

@@ -4,7 +4,7 @@ const { getRealEstateTypeEnum } = require('./enums');
 const { getRegionName, getMunicipalityName } = require('./codes');
 const { allRERequestByUiid } = require('./db/dbHelper');
 var AWS = require('aws-sdk');
-const TEMPLATE_NAME = "MarketAlertTemplate"
+const TEMPLATE_NAME = "MarketAlertTemplateDevelopment"
 
 AWS.config.update({
   region: process.env.AMAZON_REGION,
@@ -118,7 +118,8 @@ const sendBulkEmail = async (marketAlerts) => {
         requestDataValues[RERequest.uniqueId] = {
           realEstateType: RERequest.realEstateType,
           region: RERequest.region,
-          municipality: RERequest.municipality
+          municipality: RERequest.municipality,
+          requestUrl :  `${process.env.APP_URL}/nekretnine/${RERequest.uniqueId}`
         };
     });
 
@@ -129,6 +130,8 @@ const sendBulkEmail = async (marketAlerts) => {
         realEstateType: requestDataValues[marketAlert.request].realEstateType,
         municipality: requestDataValues[marketAlert.request].municipality,
         region: requestDataValues[marketAlert.request].region,
+        requestUrl: requestDataValues[marketAlert.request].requestUrl
+        
       }
 
       if (!groupedRERequests[marketAlert.request]) {
@@ -150,8 +153,9 @@ const sendBulkEmail = async (marketAlerts) => {
       const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title;
       const region = getRegionName(marketAlert.requestObject.region);
       const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality);
+      const requestUrl = marketAlert.requestObject.requestUrl
 
-      let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }`
+      let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}", "requestUrl":"${requestUrl}" }`
 
       destinations.push({
         Destination: {
@@ -207,12 +211,13 @@ const getNotificationEmailHtml = () => {
   <div>
   <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/> 
   <div/>
+  <div>Kompletan spisan nekretnina mozete pegledati ovdije: <a href="{{requestUrl}}">Nekretnine</a> <div>
   </div>`
 }
 
 const getNotificationEmailText = () => {
   return ` Zdravo, 
-  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
+  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}} , Kompletan spisan nekretnina mozete pegledati ovdije: {{requestUrl}}`
 }
 
 const createMarketAlertEmailTemplate = async () => {

app/helpers/crawlers/olxClawler.js

@@ -7,10 +7,8 @@ const Promise = require("bluebird");
 
 module.exports = class OlxCrawler {
     //TODO figure best way to handle paging
-    constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
+    constructor(hrefs = []) {
-        this.fromPage = fromPage;
+        this.hrefs = hrefs;
-        this.toPage = toPage;
-        this.maxResults = maxResults;
     }
 
     async indexPages(urls) {
@@ -36,8 +34,7 @@ module.exports = class OlxCrawler {
         const urls = this.createRequestUrls(realestateRequests);
         let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
         console.log("Final crawler results");
-        // console.log(results);
+        const flatResults = results.flat();
-        const flatResults = results.flat(); 
         console.log(flatResults);
         if (flatResults) {
             console.log(flatResults.length);
@@ -48,7 +45,7 @@ module.exports = class OlxCrawler {
                     if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
                         const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
 
-                    
+
                         if (pointInsideBoundingBox[0].length !== 0) {
                             finalResult.hasLocation = true
                             filteredResults.push(finalResult);
@@ -81,7 +78,8 @@ module.exports = class OlxCrawler {
             const olxUrl = {
                 url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`,
                 email: request.email,
-                uuid: request.uniqueId
+                uuid: request.uniqueId,
+                hrefs: this.hrefs
             }
             console.log(olxUrl.url);
             urls.push(olxUrl);
@@ -168,6 +166,7 @@ class Indexer {
     }
 
     prepareIndexers(pageNr) {
+
         console.log("Entering prepareIndexers : page nr - " + pageNr);
         const indexers = [];
         let lastPageNumber;
@@ -177,7 +176,8 @@ class Indexer {
                 const newOlxUrl = {
                     url: this.olxUrl.url.replace(/\d+$/, "") + index,
                     email: this.olxUrl.email,
-                    uuid: this.olxUrl.uuid
+                    uuid: this.olxUrl.uuid,
+                    hrefs: this.olxUrl.hrefs
                 }
                 indexers.push(new Indexer(newOlxUrl));
 
@@ -188,7 +188,8 @@ class Indexer {
                 const newOlxUrl = {
                     url: this.olxUrl.url + index,
                     email: this.olxUrl.email,
-                    uuid: this.olxUrl.uuid
+                    uuid: this.olxUrl.uuid,
+                    hrefs: this.olxUrl.hrefs
                 }
                 indexers.push(new Indexer(newOlxUrl));
             }
@@ -207,7 +208,8 @@ class Indexer {
                 const newOlxUrl = {
                     url: href,
                     email: results.olxUrl.email,
-                    uuid: results.olxUrl.uuid
+                    uuid: results.olxUrl.uuid,
+                    hrefs: this.olxUrl.hrefs
                 }
 
                 indexers.push(new Indexer(newOlxUrl));
@@ -220,11 +222,11 @@ class Indexer {
 
                 if (result !== null && result.hasOwnProperty('hrefs')) {
                     result.hrefs.forEach(href => {
-                        // console.log(href);
                         const newOlxUrl = {
                             url: href,
                             email: result.olxUrl.email,
-                            uuid: result.olxUrl.uuid
+                            uuid: result.olxUrl.uuid,
+                            hrefs: this.olxUrl.hrefs
                         }
 
                         indexers.push(new Indexer(newOlxUrl));
@@ -279,14 +281,14 @@ class Indexer {
                 return {}
             }
 
-            if (global.hrefs) {
+            // if (global.hrefs) {
 
-                if (global.hrefs[this.olxUrl.uuid] && global.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
+            if (this.olxUrl.hrefs[this.olxUrl.uuid] && this.olxUrl.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
 
-                    console.log("We found duplicate URL");
+                console.log("We found duplicate URL");
-                    return null
+                return null
-                }
             }
+            // }
 
             const res = await fetch(this.olxUrl.url);
             const body = await res.text();

app/helpers/db/dbHelper.js

@@ -50,6 +50,25 @@ const allMarketAlerts = async (fetchAll, notified) => {
 
     return await db.MarketAlert.findAll(queryObject);
   }
+
+  /**
+ * Find all , MarketAlerts depending on request
+ * 
+ * @param request string
+ * 
+ * @returns array of MarketAlerts
+ */
+const allMarketAlertsByRequest = async (request) => {
+
+  let queryObject = {
+    where : {
+      request: request
+    }
+  }
+
+  return await db.MarketAlert.findAll(queryObject);
+}
+
   
 /**
  * Find all unnotified marketalerts 
@@ -66,5 +85,6 @@ module.exports = {
   allRERequest,
   allMarketAlerts,
   allRERequestByUiid,
-  findPointInsideBoundingBox
+  findPointInsideBoundingBox,
+  allMarketAlertsByRequest
 };

app/services/crawlerService.js

@@ -4,11 +4,6 @@ const OlxCrawler = require("../helpers/crawlers/olxClawler");
 const db = require("../models/index");
 const { allMarketAlerts } = require('../helpers/db/dbHelper');
 
-const olxCrawler = new OlxCrawler(1, 2, 3);
-
-const crawlers = [
-    olxCrawler,
-];
 
 async function crawlAll() {
     console.log("CRAWLER SERVICE: crawlAll");
@@ -16,7 +11,7 @@ async function crawlAll() {
     try {
         const marketAlertsFromDb = await allMarketAlerts(true);
         const hrefs = [];
-       
+
         marketAlertsFromDb.map(marketAlert => {
             if (hrefs[marketAlert.request] === undefined) {
                 hrefs[marketAlert.request] = []
@@ -25,64 +20,66 @@ async function crawlAll() {
             hrefs[marketAlert.request].push(marketAlert.url);
         })
 
-        global.hrefs = hrefs;
         console.log("CRAWLER SERVICE: GLOBAL HREFS");
-        console.log(global.hrefs);
+        console.log(hrefs);
+        const olxCrawler = new OlxCrawler(hrefs);
 
-    } catch (e) {
+        const crawlers = [
-        console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
+            olxCrawler,
-    }
+        ];
-
+ 
-    return Promise.map(crawlers, function (crawler) {
+        return Promise.map(crawlers, function (crawler) {
-        return crawler.crawl();
+            return crawler.crawl();
-    }).then(async (results) => {
+        }).then(async (results) => {
-
-        try {
-
-            const marketAlertsFromDb = await allMarketAlerts(false, true);
-
-            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
-
-            const marketAlerts = [];
-            const mergedResults = [].concat.apply([], results);
-
-            for (const result of mergedResults) {
-                marketAlerts.push({
-                    url: result.url,
-                    realestateOrigin: "OLX",
-                    originId: 1,
-                    size: result.size,
-                    price: result.price,
-                    email: result.email,
-                    request: result.uuid,
-                    // lastDate: DataTypes.STRING,
-                    municipality: result.municipality,
-                    region: result.region,
-                    gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
-                    realEstateType: result.realEstateType,
-                    title: result.title,
-                    notified: false,
-                    hasLocation: result.hasLocation
-                })
-            }
-            console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
 
             try {
 
-                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => { 
+                const marketAlertsFromDb = await allMarketAlerts(false, true);
 
-                    return (elem.url === url && elem.request === request) }));
+                console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
-                console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
 
-                await db.MarketAlert.bulkCreate(filteredMarketAlerts);
+                const marketAlerts = [];
+                const mergedResults = [].concat.apply([], results);
 
+                for (const result of mergedResults) {
+                    marketAlerts.push({
+                        url: result.url,
+                        realestateOrigin: "OLX",
+                        originId: 1,
+                        size: result.size,
+                        price: result.price,
+                        email: result.email,
+                        request: result.uuid,
+                        municipality: result.municipality,
+                        region: result.region,
+                        gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
+                        realEstateType: result.realEstateType,
+                        title: result.title,
+                        notified: false,
+                        hasLocation: result.hasLocation
+                    })
+                }
+                console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
+
+                try {
+
+                    const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => {
+
+                        return (elem.url === url && elem.request === request)
+                    }));
+                    console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
+
+                    await db.MarketAlert.bulkCreate(filteredMarketAlerts);
+
+                } catch (e) {
+                    console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
+                }
             } catch (e) {
-                console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
+                console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
             }
-        } catch (e) {
+        })
-            console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
+    } catch (e) {
-        }
+        console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
-    })
+    }
 };
 module.exports = crawlAll;
-// crawlAll();

app/views/realEstates.ejs

@@ -6,7 +6,7 @@
       <% for(const realEstate of realEstates) { %>
       <li class="collection-item">
         <div><%= realEstate.title %>
-          <a href="realEstate.url" class="secondary-content">
+          <a href="<%= realEstate.url %>" class="secondary-content">
             <i class="material-icons">send</i>
           </a>
         </div>

index.js

@@ -10,6 +10,7 @@ const { getQuerySubmit, postQuerySubmit } = require('./app/controllers/querySubm
 const { getGoAgain } = require('./app/controllers/goAgain');
 const { getNeighborhood, postNeighborhood } = require('./app/controllers/neighborhoodMap');
 const { getUnsubscribe } = require('./app/controllers/unsubscribe');
+const { getRealEstates} = require('./app/controllers/realEstates');
 const schedule = require('node-schedule');
 const crawlAll = require('./app/services/crawlerService')
 const processNotifications = require('./app/services/notificationService')
@@ -154,6 +155,8 @@ app.get('/odjava/:request_id', getUnsubscribe);
 
 app.get('/ponovo', getGoAgain);
 
+app.get('/nekretnine/:request_id', getRealEstates);
+
 app.use('/assets', express.static('./app/public'));
 
 app.listen(port, () => console.log(`Example app listening on port ${port}!`));