experiment

2019-07-04 10:57:56 +02:00
11 changed files with 245 additions and 295 deletions
--- a/app/helpers/awsEmail.js
+++ b/app/helpers/awsEmail.js
@@ -1,10 +1,9 @@

-const dotenv = require('dotenv').config();
+const dotenv = require('dotenv');
+dotenv.config();
 const { getRealEstateTypeEnum } = require('./enums');
 const { getRegionName, getMunicipalityName } = require('./codes');
-var AWS = require('aws-sdk');
-const TEMPLATE_NAME = "MarketAlertTemplate"
-
+const AWS = require('aws-sdk');
 AWS.config.update({
  region: process.env.AMAZON_REGION,
  credentials:
@@ -28,11 +27,11 @@ const sendTemplatedEmail = async (email, request) => {
      Body: { /* required */
        Html: {
          Charset: "UTF-8",
-          Data: getGreetingsEmailHTML(request)
+          Data: getEmailHTML(request)
        },
        Text: {
          Charset: "UTF-8",
-          Data: getGreetingsEmaiTextVersion(request)
+          Data: getEmaiTextVersion(request)
        }
      },
      Subject: {
@@ -50,7 +49,7 @@ const sendTemplatedEmail = async (email, request) => {
  await sendEmailPromise;
 }

-const getGreetingsEmailHTML = (realestateRequest) => {
+const getEmailHTML = (realestateRequest) => {
  const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
  const gardenSize = realEstateType.hasGardenSize ? `<div><strong>Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2  </strong></div>` : ``

@@ -76,15 +75,15 @@ Javimi tim.

 }

-const getGreetingsEmaiTextVersion = (realestateRequest) => {
+const getEmaiTextVersion = (realestateRequest) => {
  const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
-  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : ""
+  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax  : ""

  const text = "Zdravo, \n Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima \n Ovo je tražena nekretnina: \n , Tip nekretnine: "
    + realestateRequest.realEstateType + "\n Područje" + getRegionName(realestateRequest.region) + "\n Mjesto " + getMunicipalityName(realestateRequest.region, realestateRequest.municipality)
    + "\n Kvadratura nekretnine Od " + realestateRequest.sizeMin + " do " + realestateRequest.sizeMaX +
    + gardenSize
-  "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
+    "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
    "\n Ako želis prestati dobijati obavještenja za ovu pretragu klikni" + process.env.APP_URL + "/odjava/" + realestateRequest.uniqueId +
    "\n Ako želiš promijeniti uslove pretrage klikni " + process.env.APP_URL + "/odpregled/" + realestateRequest.uniqueId +
    "\n Tvoj,\n Javimi tim"
@@ -92,111 +91,6 @@ const getGreetingsEmaiTextVersion = (realestateRequest) => {
  return text;
 }

-const sendBulkEmail = async (marketAlerts) => {
-
-  try {
-
-    destinations = []
-    groupedEmails = [];
-
-    marketAlerts.forEach(marketAlert => {
-      if (!groupedEmails[marketAlert.email]) {
-        groupedEmails[marketAlert.email] = [];
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
-      } else {
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
-      }
-    });
-
-    for (email in groupedEmails) {
-
-      const url = groupedEmails[email];
-      let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
-
-      destinations.push({
-        Destination: {
-          ToAddresses: [
-            email
-          ]
-        },
-        ReplacementTemplateData: repData
-      })
-
-    }
-    console.log("AWS EMAIL : Bulk email replacement data:");
-    console.log(destinations);
-
-    var params = {
-      Destinations:
-        destinations,
-      Source: process.env.SOURCE_EMAIL, /* required */
-      Template: TEMPLATE_NAME, /* required */
-      DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }',
-      ReplyToAddresses: [
-        process.env.SOURCE_EMAIL,
-      ]
-    };
-
-    // Create the promise and SES service object
-    const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise();
-    const awsResult = await sendPromise;
-    console.log("AWS SES bulk email response");
-    console.log(awsResult);
-
-
-  } catch (e) {
-    console.log("Could not send bulk email", e)
-  }
-
-
-}
-
-const toAWSArray = (urlArray) => {
-  let arrayString = ""
-  urlArray.forEach(element => {
-    const formatetdTitle = element.title.replace(/"/g, "");
-    arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
-  });
-
-  return arrayString.slice(0, -1);
-}
-
-const getNotificationEmailHtml = () => {
-  return `<h2> Zdravo, 
-  Pronašli smo nekretninu koju ste tražili. </h2>
-  <h3> Ovo su tražene nekretnine:  </h3> 
-  <div>
-  <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/> 
-  <div/>
-  </div>`
-}
-
-const getNotificationEmailText = () => {
-  return ` Zdravo, 
-  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
-}
-
-const createMarketAlertEmailTemplate = async () => {
-  const marketAlertTemplate = {
-    Template: {
-      TemplateName: "MarketAlertTemplate",
-      SubjectPart: "Javi mi obavijest",
-      TextPart: getNotificationEmailText(),
-      HtmlPart: getNotificationEmailHtml()
-    }
-  }
-
-  try {
-    const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise();
-    await templatePromise
-
-  } catch (e) {
-    console.log("Could not create MarketAlertEmailTemplate", e);
-  }
-}
-
 module.exports = {
-  sendTemplatedEmail,
-  sendBulkEmail,
-  createMarketAlertEmailTemplate
+  sendTemplatedEmail
 };
--- a/app/helpers/crawlers/olxClawler.js
+++ b/app/helpers/crawlers/olxClawler.js
@@ -1,6 +1,6 @@
 const fetch = require('node-fetch');
 const cheerio = require('cheerio');
-const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
+const { allRERequest, findPointInsideBoundingBox } = require('../url');
 const { getRealEstateTypeEnum } = require('../enums');
 const { getRegion, getMunicipality } = require('../codes')
 const Promise = require("bluebird");
@@ -27,7 +27,7 @@ module.exports = class OlxCrawler {
            // }

            //TODO remove properties that are not needed, and add some if they are missing
-            const title = $('#naslovartikla').text().trim();
+            const title = $('#naslovartikla').text();
            const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();

            const price = $('#pc > p:nth-child(2)').text();
@@ -87,14 +87,14 @@ module.exports = class OlxCrawler {

            const data = {
                realEstateType: this.getCategoryId(realEstateType),
-                email: email,
+                email : email,
                olxId: olxId,
                // category: category,
                url,
                title,
-                price: isNaN(parsedPrice) ? 0 : parsedPrice,
+                price: isNaN(parsedPrice) ? price : parsedPrice,
                size: parseFloat(size),
-                gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
+                gardenSize: parseFloat(gardenSize),
                address,
                region,
                municipality,
@@ -118,7 +118,6 @@ module.exports = class OlxCrawler {

    async indexPage(olxUrl, maxResults = 1000) {
        try {
-            //TODO fix paging
            // console.log('Starting to index page: ' + pageNr);
            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;

@@ -143,6 +142,7 @@ module.exports = class OlxCrawler {
                if (singleData) {
                    results.push(singleData);
                }
+                // await this.sleep(500);
            }

            return results;
@@ -151,64 +151,69 @@ module.exports = class OlxCrawler {
        }
    }

-    getCategoryId(category) {
-
-        switch (category) {
-            case 'Stanovi':
-                return 'stan';
-
-            case 'Vikendice':
-                return 'vikendica'
-
-            case 'Kuće':
-                return 'kuca';
-
-            default:
-                return '';
+    getCategoryId (category) {
+        if (category === 'Stanovi') {
+          return 'stan';
+        } else if (category === 'Vikendice') {
+          return 'vikendica';
+        } else if (category === 'Kuće') {
+          return 'kuca';
        }
-    }
+      }

    async indexPages(urls, start, end, maxResults = 1000) {
-        //TODO fix paging
        // let results = {};
        // for (let i = start; i <= end; i++) {
        //     let result = await this.indexPage(i, maxResults);
        //     Object.assign(results, result)
+        //     await this.sleep(5000);
        // }
        // return results;

        let results = [];
+        const indexers= [];
+        let it = 3
        for (let url of urls) {
-            let result = await this.indexPage(url, maxResults);
-            results.push(result);
+            // let result = await this.indexPage(url, maxResults);
+            // results.push(result);
+            it++
+            indexers.push(new Indexer(it * 2000));
+
        }
-        return results;
+        return Promise.map(indexers, function (indexer) {
+            return indexer.indexPage();
+        }).then(async (results) => {
+            return results
+        })       
    }

    async crawl() {
-        console.log("OLX CRAWLER: start crawl");

        const filteredResults = [];
-        const realestateRequests = await allRERequest();
-        console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
+        const realestateRequests = await allRERequest()
        const urls = this.createRequestUrls(realestateRequests);
        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);

        for (const result of results) {
-            for (const finalResult of result) {
-                if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
-                    const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
+            for (const re1 of result) {
+                if (re1.lat !== undefined && re1.lat !== null && re1.lat !== "") {
+                    const pointInsideBoundingBox = await findPointInsideBoundingBox([re1.lng, re1.lat]);

                    if (pointInsideBoundingBox[0].length !== 0) {
-                        filteredResults.push(finalResult);
+                        filteredResults.push(re1);
                    }
                }
            }
        }
-        console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
+        // await this.sleep(10000);
+        console.log(filteredResults);
        return filteredResults;
    }

+    async sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+      }
+
    createRequestUrls(realestateRequests) {
        const urls = []

@@ -222,7 +227,7 @@ module.exports = class OlxCrawler {
            const priceMax = "do=" + request.priceMax;

            const olxUrl = {
-                url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
+                url: "https://www.olx.ba/pretraga?" + realsestateType + "&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&" + region + "&" + municipality + "&" + priceMin + "&" + priceMax + "&vrsta=samoprodaja&" + sizeMin + "&" + sizeMax,
                email: request.email
            }
            console.log(olxUrl.url);
@@ -233,3 +238,170 @@ module.exports = class OlxCrawler {
    }
 };

+
+class Indexer {
+    
+    constructor(olxUrl, email, uuid) {
+        this.olxUrl = olxUrl
+        this.email = email
+        this.uuid = uuid
+    }
+
+    async indexPage() {
+
+        try {
+            // console.log('Starting to index page: ' + pageNr);
+            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+
+            // const res = await fetch(this.olxUrl.url);
+            // const body = await res.text();
+            // const $ = cheerio.load(body);
+            // const hrefs = [];
+            // const results = [];
+        
+            // $('#rezultatipretrage').find('.listitem').each((i, elem) => {
+            //     const href = $(elem).find('a').first().attr('href');
+            //     hrefs.push(href);
+            // });
+
+            // let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
+
+            // for (let i = 0; i < hrefs.length; i++) {
+            //     console.log(`indexing: ${hrefs[i]}`);
+
+            //     const singleData = await this.indexSingle(hrefs[i], this.olxUrl.email);
+
+            //     if (singleData) {
+            //         results.push(singleData);
+            //     }
+            //     // await this.sleep(500);
+            // }
+            await this.sleep(this.olxUrl);
+            console.log('Finished indexing PAGE');
+            const singleIndex = [new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl)]
+            return Promise.map(singleIndex, function (indexer) {
+                return indexer.indexSingle();
+            }).then(async (results) => {
+               
+                return results
+            })  
+
+
+          
+            // return results;
+        } catch (e) {
+            console.error('Exception caught:' + e);
+        }
+    }
+
+    async indexSingle() {
+        // try {
+        //     const res = await fetch(url);
+        //     const body = await res.text();
+        //     const $ = cheerio.load(body);
+
+        //     //TODO figure out what to do with username
+        //     const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+        //     // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+        //     //     return null;
+        //     // }
+
+        //     //TODO remove properties that are not needed, and add some if they are missing
+        //     const title = $('#naslovartikla').text();
+        //     const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
+
+        //     const price = $('#pc > p:nth-child(2)').text();
+        //     const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
+        //     const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
+        //     const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
+        //     const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
+        //     const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+        //     const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+        //     const time = $('time').attr('datetime');
+        //     const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+        //     const descriptions = $('.artikal_detaljniopis_tekst');
+        //     // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
+        //     const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+        //     const imgRe = /href":("[^"]*")/g;
+        //     const matches = latLngRe.exec(body);
+        //     let lng = '',
+        //         lat = '';
+
+
+        //     const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+        //     const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
+        //     // TODO we dont save images ??
+
+        //     // const images = [];
+        //     // const imgMatches = body.match(imgRe);
+
+        //     // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+        //     //     let img = imgMatches[i].replace("href\":", "")
+        //     //     img = img.replace("\"", "");
+        //     //     img = img.replace("\"", "");
+        //     //     images.push(img);
+        //     // }
+
+        //     // const uploadPromises = images.map(img => {
+        //     //     const imgFixed = eval(`'${img}'`);
+        //     //     return cloudinary.uploader.upload(eval(`'${img}'`));
+        //     // });
+
+        //     // const uploadResults = await Promise.all(uploadPromises);
+        //     // const cloudinaryImages = uploadResults.map(ur => ur.url);
+
+        //     if (matches && matches.length >= 3) {
+        //         lat = matches[1];
+        //         lng = matches[2];
+        //     }
+
+        //     const parsedPrice = parsePrice(price);
+
+        //     const locationArray = location.split(",");
+        //     const region = locationArray[0];
+        //     const municipality = locationArray[1];
+
+        //     const data = {
+        //         realEstateType: this.getCategoryId(realEstateType),
+        //         email : email,
+        //         olxId: olxId,
+        //         // category: category,
+        //         url,
+        //         title,
+        //         price: isNaN(parsedPrice) ? price : parsedPrice,
+        //         size: parseFloat(size),
+        //         gardenSize: parseFloat(gardenSize),
+        //         address,
+        //         region,
+        //         municipality,
+        //         // adType: AD_TYPE_SALE,
+        //         time,
+        //         shortDescription: descriptions.first().text(),
+        //         longDescription: descriptions.last().text(),
+        //         lat,
+        //         lng,
+        //         loc: [parseFloat(lat), parseFloat(lng)],
+        //         // images: cloudinaryImages
+        //     };
+
+        //     return data;
+        // } catch (e) {
+        //     console.error('Exception caught: ' + e.message);
+        // }
+
+        // return null;
+        await this.sleep(this.olxUrl);
+        console.log("Finished indexing single page");
+        return {};
+    }
+
+    async sleep(ms) {
+        console.log(ms);
+        return new Promise(resolve => setTimeout(resolve, ms));
+      }
+}
--- a/app/helpers/crawlers/olxPagerIndex.js
+++ b/app/helpers/crawlers/olxPagerIndex.js
--- a/app/helpers/db/dbHelper.js
+++ b/app/helpers/db/dbHelper.js
@@ -1,54 +1,10 @@
-const db = require('../../models/index');

-/**
- * Find all subscribed RealEstateRequests 
- */
-const allRERequest = async () => {
-    return await db.RealEstateRequest.findAll({
-        where: {
-            subscribed: true
-        }
-      });
-}
+// const db = require('../../models/index');

-/**
- * Find all , or all depending on notified bolean marketalerts, and order them by email
- * 
- * @param fechAll bolean
- * @param notified bolean
- * 
- * @returns array of MarketAlerts
- */
-const allMarketAlerts = async (fetchAll, notified) => {

-    let queryObject = {
-        order: [
-            ['email', 'DESC'],
-        ]
-    }
+// const bulkInsert = async (reuslts) => {
+//     db.MarketAlert.bulkCreate({

-    if (!fetchAll){
-        queryObject.where = {
-            notified: notified
-        }
-    }
-
-    return await db.MarketAlert.findAll(queryObject);
-  }
-  
-/**
- * Find all unnotified marketalerts 
- *  @param latLng array
- *  @param email string
- * 
- * @returns array of MarketAlerts
- */
-const findPointInsideBoundingBox = async (latLng, email) => {
-  return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
-}
-
-module.exports = {
-  allRERequest,
-  allMarketAlerts,
-  findPointInsideBoundingBox
-};
+//     })
+   
+//   }
--- a/app/helpers/url.js
+++ b/app/helpers/url.js
@@ -7,6 +7,17 @@ const currentRERequest = async (req) => {
  const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
  return request;
 };
+// TODO Fetch only subscribed realestate requests
+const allRERequest = async () => {
+  return await db.RealEstateRequest.findAll();
+}
+
+const findPointInsideBoundingBox = async (latLng) => {
+  return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
+}
+
 module.exports = {
  currentRERequest,
+  allRERequest,
+  findPointInsideBoundingBox
 };
--- a/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js
+++ b/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js
@@ -1,20 +0,0 @@
-'use strict';
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'notified',
-        {
-          type: Sequelize.BOOLEAN
-        }
-    );
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'notified'
-    );
-  }
-};
--- a/app/migrations/20190628165512-add-title-to-marketalerts.js
+++ b/app/migrations/20190628165512-add-title-to-marketalerts.js
@@ -1,20 +0,0 @@
-'use strict';
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'title',
-        {
-          type: Sequelize.STRING
-        }
-    );
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'title'
-    );
-  }
-};
--- a/app/models/marketalert.js
+++ b/app/models/marketalert.js
@@ -11,8 +11,6 @@ module.exports = (sequelize, DataTypes) => {
    municipality : DataTypes.STRING,
    region : DataTypes.STRING,
    realEstateType : DataTypes.STRING,
-    notified : DataTypes.BOOLEAN,
-    title : DataTypes.STRING,
    
    email: { 
      type: DataTypes.STRING,
--- a/app/services/crawlerService.js
+++ b/app/services/crawlerService.js
@@ -2,16 +2,18 @@
 const Promise = require("bluebird");
 const OlxCrawler = require("../helpers/crawlers/olxClawler");
 const db = require("../models/index");
-const { allMarketAlerts } = require('../helpers/db/dbHelper');

 const olxCrawler = new OlxCrawler(1, 2, 3);
+const olxCrawler1 = new OlxCrawler(1, 2, 3);
+const olxCrawler2 = new OlxCrawler(1, 2, 3);

 const crawlers = [
    olxCrawler,
+    olxCrawler1,
+    olxCrawler2,
 ];

 async function crawlAll() {
-    console.log("CRAWLER SERVICE: crawlAll");

    Promise.map(crawlers, function (crawler) {
        return crawler.crawl();
@@ -19,8 +21,7 @@ async function crawlAll() {

        try {

-            const marketAlertsFromDb = await allMarketAlerts(true);
-            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
+            const marketAlertsFromDb = await db.MarketAlert.findAll();

            const marketAlerts = [];
            const mergedResults = [].concat.apply([], results);
@@ -37,30 +38,21 @@ async function crawlAll() {
                    municipality: result.municipality,
                    region: result.region,
                    gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
-                    realEstateType: result.realEstateType,
-                    title: result.title,
-                    notified: false
+                    realEstateType: result.realEstateType
                })
            }
-            console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
-
            try {
-
-                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
-                console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
-
+                const filteredMarketAlerts =  marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
                await db.MarketAlert.bulkCreate(filteredMarketAlerts);
-                process.exit();
-
+                process.exit()
            } catch (e) {
-                console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
-                process.exit();
+                console.log("Could not bulkCreate marketalers reason: ", e);
            }
        } catch (e) {
-            console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
-            process.exit();
+            console.log("Error crawling. Trying next crawler! ", e);
        }
    })
 };

 crawlAll();
+
--- a/app/services/notificationService.js
+++ b/app/services/notificationService.js
@@ -1,32 +0,0 @@
-
-const Promise = require("bluebird");
-const db = require("../models/index");
-const { allMarketAlerts } = require('../helpers/db/dbHelper');
-const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/awsEmail');
-
-
-async function processNotifications() {
-
-    try {
-        const marketAlerts = await allMarketAlerts(false, false);
-        console.log(marketAlerts.length)
-        await createMarketAlertEmailTemplate();
-        if (marketAlerts.length > 0) {
-            console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length)
-            await sendBulkEmail(marketAlerts);
-        } else {
-            console.log("NOTIFICATION SERVICE: No new alerts");
-            return;
-        }
-
-        await db.MarketAlert.update(
-            { notified: true }, /* set attributes' value */
-            { where: { notified: false } } /* where criteria */
-        );
-        process.exit();
-    } catch (e) {
-        console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
-    }
-}
-
-processNotifications();
--- a/package.json
+++ b/package.json
@@ -7,8 +7,7 @@
    "test": "echo \"Error: no test specified\" && exit 1",
    "start": "node ./index.js",
    "start-mon": "nodemon ./index.js",
-    "crawler": "node ./app/services/crawlerService.js",
-    "notification": "node ./app/services/notificationService.js",
+    "scheduler": "node ./app/services/crawlerService.js",
    "migrate": "cd app && npx sequelize db:migrate",
    "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
    "docker-start": "docker start pg_marketalerts",