Merge branch 'marketalerts-page' into 'master'

Real Estate Page See merge request saburly/marketalarm/web!24
2019-07-15 09:48:08 +00:00
parent 4517624fa8 1999d45cb2
commit 09792db21c
7 changed files with 159 additions and 82 deletions
--- a/app/controllers/realEstates.js
+++ b/app/controllers/realEstates.js
@@ -0,0 +1,18 @@
+
+const {allMarketAlertsByRequest} = require('../helpers/db/dbHelper');
+
+const getRealEstates = async (req,res) => {
+    console.log("Enter get realestates");
+    const request = req.params['request_id'];
+    console.log(req.params['request_id']);
+    const realEstates = await allMarketAlertsByRequest(request);
+    console.log(realEstates);
+
+    const title = "Ovo su nekretnine koje smo pronašli za vas"
+    res.render('realEstates', {realEstates, title } );
+  };
+  
+  module.exports = {
+    getRealEstates
+  };
+  
--- a/app/helpers/awsEmail.js
+++ b/app/helpers/awsEmail.js
@@ -109,16 +109,17 @@ const sendBulkEmail = async (marketAlerts) => {
      return { uniqueId: marketAlert }
    });

-    const RERequest = await allRERequestByUiid(RERequestUuids);
+    const RERequests = await allRERequestByUiid(RERequestUuids);
    const requestDataValues = [];

-    RERequest.forEach(RERequest => {
+    RERequests.forEach(RERequest => {
      var formatedRequest = {};
      formatedRequest[RERequest.uniqueId] =
        requestDataValues[RERequest.uniqueId] = {
          realEstateType: RERequest.realEstateType,
          region: RERequest.region,
-          municipality: RERequest.municipality
+          municipality: RERequest.municipality,
+          requestUrl :  `${process.env.APP_URL}/nekretnine/${RERequest.uniqueId}`
        };
    });

@@ -129,6 +130,8 @@ const sendBulkEmail = async (marketAlerts) => {
        realEstateType: requestDataValues[marketAlert.request].realEstateType,
        municipality: requestDataValues[marketAlert.request].municipality,
        region: requestDataValues[marketAlert.request].region,
+        requestUrl: requestDataValues[marketAlert.request].requestUrl
+        
      }

      if (!groupedRERequests[marketAlert.request]) {
@@ -150,8 +153,9 @@ const sendBulkEmail = async (marketAlerts) => {
      const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title;
      const region = getRegionName(marketAlert.requestObject.region);
      const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality);
+      const requestUrl = marketAlert.requestObject.requestUrl

-      let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }`
+      let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}", "requestUrl":"${requestUrl}" }`

      destinations.push({
        Destination: {
@@ -207,12 +211,13 @@ const getNotificationEmailHtml = () => {
  <div>
  <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/> 
  <div/>
+  <div>Kompletan spisak nekretnina možete pegledati ovdije: <a href="{{requestUrl}}">Nekretnine</a> <div>
  </div>`
 }

 const getNotificationEmailText = () => {
  return ` Zdravo, 
-  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
+  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}} , Kompletan spisan nekretnina mozete pegledati ovdije: {{requestUrl}}`
 }

 const createMarketAlertEmailTemplate = async () => {
--- a/app/helpers/crawlers/olxClawler.js
+++ b/app/helpers/crawlers/olxClawler.js
@@ -7,10 +7,8 @@ const Promise = require("bluebird");

 module.exports = class OlxCrawler {
    //TODO figure best way to handle paging
-    constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
-        this.fromPage = fromPage;
-        this.toPage = toPage;
-        this.maxResults = maxResults;
+    constructor(hrefs = []) {
+        this.hrefs = hrefs;
    }

    async indexPages(urls) {
@@ -36,16 +34,18 @@ module.exports = class OlxCrawler {
        const urls = this.createRequestUrls(realestateRequests);
        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
        console.log("Final crawler results");
-        if (results[0]) {
-            console.log(results[0].length);
+        const flatResults = results.flat();
+        console.log(flatResults);
+        if (flatResults) {
+            console.log(flatResults.length);

-            for (const finalResult of results[0]) {
+            for (const finalResult of flatResults) {

                if (null !== finalResult) {
                    if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
-                        const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
+                        const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email, finalResult.uuid);
+

-                    
                        if (pointInsideBoundingBox[0].length !== 0) {
                            finalResult.hasLocation = true
                            filteredResults.push(finalResult);
@@ -78,7 +78,8 @@ module.exports = class OlxCrawler {
            const olxUrl = {
                url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`,
                email: request.email,
-                uuid: request.uniqueId
+                uuid: request.uniqueId,
+                hrefs: this.hrefs
            }
            console.log(olxUrl.url);
            urls.push(olxUrl);
@@ -165,6 +166,7 @@ class Indexer {
    }

    prepareIndexers(pageNr) {
+
        console.log("Entering prepareIndexers : page nr - " + pageNr);
        const indexers = [];
        let lastPageNumber;
@@ -174,7 +176,8 @@ class Indexer {
                const newOlxUrl = {
                    url: this.olxUrl.url.replace(/\d+$/, "") + index,
                    email: this.olxUrl.email,
-                    uuid: this.olxUrl.uuid
+                    uuid: this.olxUrl.uuid,
+                    hrefs: this.olxUrl.hrefs
                }
                indexers.push(new Indexer(newOlxUrl));

@@ -185,7 +188,8 @@ class Indexer {
                const newOlxUrl = {
                    url: this.olxUrl.url + index,
                    email: this.olxUrl.email,
-                    uuid: this.olxUrl.uuid
+                    uuid: this.olxUrl.uuid,
+                    hrefs: this.olxUrl.hrefs
                }
                indexers.push(new Indexer(newOlxUrl));
            }
@@ -204,7 +208,8 @@ class Indexer {
                const newOlxUrl = {
                    url: href,
                    email: results.olxUrl.email,
-                    uuid: results.olxUrl.uuid
+                    uuid: results.olxUrl.uuid,
+                    hrefs: this.olxUrl.hrefs
                }

                indexers.push(new Indexer(newOlxUrl));
@@ -217,11 +222,11 @@ class Indexer {

                if (result !== null && result.hasOwnProperty('hrefs')) {
                    result.hrefs.forEach(href => {
-                        // console.log(href);
                        const newOlxUrl = {
                            url: href,
                            email: result.olxUrl.email,
-                            uuid: result.olxUrl.uuid
+                            uuid: result.olxUrl.uuid,
+                            hrefs: this.olxUrl.hrefs
                        }

                        indexers.push(new Indexer(newOlxUrl));
@@ -276,14 +281,14 @@ class Indexer {
                return {}
            }

-            if (global.hrefs) {
+            // if (global.hrefs) {

-                if (global.hrefs[this.olxUrl.uuid] && global.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
+            if (this.olxUrl.hrefs[this.olxUrl.uuid] && this.olxUrl.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {

-                    console.log("We found duplicate URL");
-                    return null
-                }
+                console.log("We found duplicate URL");
+                return null
            }
+            // }

            const res = await fetch(this.olxUrl.url);
            const body = await res.text();
--- a/app/helpers/db/dbHelper.js
+++ b/app/helpers/db/dbHelper.js
@@ -50,6 +50,25 @@ const allMarketAlerts = async (fetchAll, notified) => {

    return await db.MarketAlert.findAll(queryObject);
  }
+
+  /**
+ * Find all , MarketAlerts depending on request
+ * 
+ * @param request string
+ * 
+ * @returns array of MarketAlerts
+ */
+const allMarketAlertsByRequest = async (request) => {
+
+  let queryObject = {
+    where : {
+      request: request
+    }
+  }
+
+  return await db.MarketAlert.findAll(queryObject);
+}
+
  
 /**
 * Find all unnotified marketalerts 
@@ -58,13 +77,14 @@ const allMarketAlerts = async (fetchAll, notified) => {
 * 
 * @returns array of MarketAlerts
 */
-const findPointInsideBoundingBox = async (latLng, email) => {
-  return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
+const findPointInsideBoundingBox = async (latLng, email, uniqueId) => {
+  return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND "uniqueId" = '${uniqueId}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
 }

 module.exports = {
  allRERequest,
  allMarketAlerts,
  allRERequestByUiid,
-  findPointInsideBoundingBox
+  findPointInsideBoundingBox,
+  allMarketAlertsByRequest
 };
--- a/app/services/crawlerService.js
+++ b/app/services/crawlerService.js
@@ -4,11 +4,6 @@ const OlxCrawler = require("../helpers/crawlers/olxClawler");
 const db = require("../models/index");
 const { allMarketAlerts } = require('../helpers/db/dbHelper');

-const olxCrawler = new OlxCrawler(1, 2, 3);
-
-const crawlers = [
-    olxCrawler,
-];

 async function crawlAll() {
    console.log("CRAWLER SERVICE: crawlAll");
@@ -16,7 +11,7 @@ async function crawlAll() {
    try {
        const marketAlertsFromDb = await allMarketAlerts(true);
        const hrefs = [];
-       
+
        marketAlertsFromDb.map(marketAlert => {
            if (hrefs[marketAlert.request] === undefined) {
                hrefs[marketAlert.request] = []
@@ -25,62 +20,66 @@ async function crawlAll() {
            hrefs[marketAlert.request].push(marketAlert.url);
        })

-        global.hrefs = hrefs;
        console.log("CRAWLER SERVICE: GLOBAL HREFS");
-        console.log(global.hrefs);
+        console.log(hrefs);
+        const olxCrawler = new OlxCrawler(hrefs);

-    } catch (e) {
-        console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
-    }
-
-    return Promise.map(crawlers, function (crawler) {
-        return crawler.crawl();
-    }).then(async (results) => {
-
-        try {
-
-            const marketAlertsFromDb = await allMarketAlerts(false, true);
-
-            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
-
-            const marketAlerts = [];
-            const mergedResults = [].concat.apply([], results);
-
-            for (const result of mergedResults) {
-                marketAlerts.push({
-                    url: result.url,
-                    realestateOrigin: "OLX",
-                    originId: 1,
-                    size: result.size,
-                    price: result.price,
-                    email: result.email,
-                    request: result.uuid,
-                    // lastDate: DataTypes.STRING,
-                    municipality: result.municipality,
-                    region: result.region,
-                    gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
-                    realEstateType: result.realEstateType,
-                    title: result.title,
-                    notified: false,
-                    hasLocation: result.hasLocation
-                })
-            }
-            console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
+        const crawlers = [
+            olxCrawler,
+        ];
+ 
+        return Promise.map(crawlers, function (crawler) {
+            return crawler.crawl();
+        }).then(async (results) => {

            try {

-                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
-                console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
+                const marketAlertsFromDb = await allMarketAlerts(false, true);

-                await db.MarketAlert.bulkCreate(filteredMarketAlerts);
+                console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);

+                const marketAlerts = [];
+                const mergedResults = [].concat.apply([], results);
+
+                for (const result of mergedResults) {
+                    marketAlerts.push({
+                        url: result.url,
+                        realestateOrigin: "OLX",
+                        originId: 1,
+                        size: result.size,
+                        price: result.price,
+                        email: result.email,
+                        request: result.uuid,
+                        municipality: result.municipality,
+                        region: result.region,
+                        gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
+                        realEstateType: result.realEstateType,
+                        title: result.title,
+                        notified: false,
+                        hasLocation: result.hasLocation
+                    })
+                }
+                console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
+
+                try {
+
+                    const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => {
+
+                        return (elem.url === url && elem.request === request)
+                    }));
+                    console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
+
+                    await db.MarketAlert.bulkCreate(filteredMarketAlerts);
+
+                } catch (e) {
+                    console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
+                }
            } catch (e) {
-                console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
+                console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
            }
-        } catch (e) {
-            console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
-        }
-    })
+        })
+    } catch (e) {
+        console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
+    }
 };
 module.exports = crawlAll;
-// crawlAll();
--- a/app/views/realEstates.ejs
+++ b/app/views/realEstates.ejs
@@ -0,0 +1,16 @@
+<!--suppress HtmlUnknownAnchorTarget -->
+<% include partials/navBar %> 
+
+<div class="row center-align">
+    <ul class="collection with-header">
+      <% for(const realEstate of realEstates) { %>
+      <li class="collection-item">
+        <div><%= realEstate.title %>
+          <a href="<%= realEstate.url %>" class="secondary-content">
+            <i class="material-icons">send</i>
+          </a>
+        </div>
+      </li>
+      <% } %>
+    </ul>
+  </div>
--- a/index.js
+++ b/index.js
@@ -10,6 +10,7 @@ const { getQuerySubmit, postQuerySubmit } = require('./app/controllers/querySubm
 const { getGoAgain } = require('./app/controllers/goAgain');
 const { getNeighborhood, postNeighborhood } = require('./app/controllers/neighborhoodMap');
 const { getUnsubscribe } = require('./app/controllers/unsubscribe');
+const { getRealEstates} = require('./app/controllers/realEstates');
 const schedule = require('node-schedule');
 const crawlAll = require('./app/services/crawlerService')
 const processNotifications = require('./app/services/notificationService')
@@ -154,6 +155,8 @@ app.get('/odjava/:request_id', getUnsubscribe);

 app.get('/ponovo', getGoAgain);

+app.get('/nekretnine/:request_id', getRealEstates);
+
 app.use('/assets', express.static('./app/public'));

 app.listen(port, () => console.log(`Example app listening on port ${port}!`));
@@ -167,3 +170,14 @@ var rule = new schedule.RecurrenceRule();
    await processNotifications();
    console.log(new Date(), 'Notification service finished');
  });
+
+ /**
+  * Add flat method to Array
+  */
+  Object.defineProperty(Array.prototype, 'flat', {
+    value: function(depth = 1) {
+      return this.reduce(function (flat, toFlatten) {
+        return flat.concat((Array.isArray(toFlatten) && (depth>1)) ? toFlatten.flat(depth-1) : toFlatten);
+      }, []);
+    }
+});