experiment

README.md

@@ -34,10 +34,3 @@ this will create and run postgres image and then execute migrations
 
 5. Run app  
 `npm start` or `npm run start-mon` to run app with automatic restart on code change
-
-
-### AWS SES
-
-    - AWS SES credentials are handled with env vratiables
-    - Notification emails are sent in batches of 50, by using SES templates
-    - Make sure that you are using different templates for different envirorments

app/helpers/awsEmail.js

@@ -1,11 +1,9 @@
 
-const dotenv = require('dotenv').config();
+const dotenv = require('dotenv');
+dotenv.config();
 const { getRealEstateTypeEnum } = require('./enums');
 const { getRegionName, getMunicipalityName } = require('./codes');
-const { allRERequestByUiid } = require('./db/dbHelper');
+const AWS = require('aws-sdk');
-var AWS = require('aws-sdk');
-const TEMPLATE_NAME = "MarketAlertTemplate"
-
 AWS.config.update({
   region: process.env.AMAZON_REGION,
   credentials:
@@ -29,16 +27,16 @@ const sendTemplatedEmail = async (email, request) => {
       Body: { /* required */
         Html: {
           Charset: "UTF-8",
-          Data: getGreetingsEmailHTML(request)
+          Data: getEmailHTML(request)
         },
         Text: {
           Charset: "UTF-8",
-          Data: getGreetingsEmaiTextVersion(request)
+          Data: getEmaiTextVersion(request)
         }
       },
       Subject: {
         Charset: 'UTF-8',
-        Data: `Javimi Potvrda: ${getSubject(request.realEstateType, request.region, request.municipality)}`
+        Data: `Javimi Potvrda: ${request.realEstateType} ${getRegionName(request.region)}, ${getMunicipalityName(request.region, request.municipality)}`
       }
     },
     Source: process.env.SOURCE_EMAIL, /* required */
@@ -51,7 +49,7 @@ const sendTemplatedEmail = async (email, request) => {
   await sendEmailPromise;
 }
 
-const getGreetingsEmailHTML = (realestateRequest) => {
+const getEmailHTML = (realestateRequest) => {
   const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
   const gardenSize = realEstateType.hasGardenSize ? `<div><strong>Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2  </strong></div>` : ``
 
@@ -77,15 +75,15 @@ Javimi tim.
 
 }
 
-const getGreetingsEmaiTextVersion = (realestateRequest) => {
+const getEmaiTextVersion = (realestateRequest) => {
   const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
-  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : ""
+  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax  : ""
 
   const text = "Zdravo, \n Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima \n Ovo je tražena nekretnina: \n , Tip nekretnine: "
     + realestateRequest.realEstateType + "\n Područje" + getRegionName(realestateRequest.region) + "\n Mjesto " + getMunicipalityName(realestateRequest.region, realestateRequest.municipality)
     + "\n Kvadratura nekretnine Od " + realestateRequest.sizeMin + " do " + realestateRequest.sizeMaX +
     + gardenSize
-  "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
+    "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
     "\n Ako želis prestati dobijati obavještenja za ovu pretragu klikni" + process.env.APP_URL + "/odjava/" + realestateRequest.uniqueId +
     "\n Ako želiš promijeniti uslove pretrage klikni " + process.env.APP_URL + "/odpregled/" + realestateRequest.uniqueId +
     "\n Tvoj,\n Javimi tim"
@@ -93,153 +91,6 @@ const getGreetingsEmaiTextVersion = (realestateRequest) => {
   return text;
 }
 
-const sendBulkEmail = async (marketAlerts) => {
-
-  try {
-
-    destinations = []
-    groupedRERequests = [];
-
-
-    const RERequestUuidsMaped = marketAlerts.map(marketAlert => marketAlert.request);
-
-    const RERequestUuidsArray = Array.from(new Set(RERequestUuidsMaped));
-
-    const RERequestUuids = RERequestUuidsArray.map(marketAlert => {
-      return { uniqueId: marketAlert }
-    });
-
-    const RERequest = await allRERequestByUiid(RERequestUuids);
-    const requestDataValues = [];
-
-    RERequest.forEach(RERequest => {
-      var formatedRequest = {};
-      formatedRequest[RERequest.uniqueId] =
-        requestDataValues[RERequest.uniqueId] = {
-          realEstateType: RERequest.realEstateType,
-          region: RERequest.region,
-          municipality: RERequest.municipality
-        };
-    });
-
-    marketAlerts.forEach(marketAlert => {
-
-      const requestObject = {
-        email: marketAlert.email,
-        realEstateType: requestDataValues[marketAlert.request].realEstateType,
-        municipality: requestDataValues[marketAlert.request].municipality,
-        region: requestDataValues[marketAlert.request].region,
-      }
-
-      if (!groupedRERequests[marketAlert.request]) {
-        groupedRERequests[marketAlert.request] = {
-          requestObject: requestObject,
-          marketAlertArray: []
-        };
-      }
-
-      groupedRERequests[marketAlert.request].marketAlertArray.push({
-        url: marketAlert.url,
-        title: marketAlert.title,
-      });
-    });
-    for (request in groupedRERequests) {
-
-      const marketAlert = groupedRERequests[request];
-      let extractedData = toAWSArray(marketAlert.marketAlertArray);
-      const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title;
-      const region = getRegionName(marketAlert.requestObject.region);
-      const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality);
-
-      let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }`
-
-      destinations.push({
-        Destination: {
-          ToAddresses: [
-            marketAlert.requestObject.email
-          ]
-        },
-        ReplacementTemplateData: repData
-      })
-
-    }
-    console.log("AWS EMAIL : Bulk email replacement data:");
-    console.log(destinations);
-
-    var params = {
-      Destinations:
-        destinations,
-      Source: process.env.SOURCE_EMAIL, /* required */
-      Template: TEMPLATE_NAME, /* required */
-      DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }',
-      ReplyToAddresses: [
-        process.env.SOURCE_EMAIL,
-      ]
-    };
-
-    // Create the promise and SES service object
-    const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise();
-    const awsResult = await sendPromise;
-    console.log("AWS SES bulk email response");
-    console.log(awsResult);
-
-
-  } catch (e) {
-    console.log("Could not send bulk email", e)
-  }
-}
-
-const toAWSArray = (urlArray) => {
-
-  let arrayString = ""
-  urlArray.forEach(element => {
-    const formatetdTitle = element.title.replace(/"/g, "");
-    arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
-  });
-  return arrayString.slice(0, -1);
-
-}
-
-const getNotificationEmailHtml = () => {
-  return `<h2> Zdravo, 
-  Pronašli smo nekretninu koju ste tražili. </h2>
-  <h3> Ovo su tražene nekretnine:  </h3> 
-  <div>
-  <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/> 
-  <div/>
-  </div>`
-}
-
-const getNotificationEmailText = () => {
-  return ` Zdravo, 
-  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
-}
-
-const createMarketAlertEmailTemplate = async () => {
-  const marketAlertTemplate = {
-    Template: {
-      TemplateName: TEMPLATE_NAME,
-      SubjectPart: "Javi mi obavijest: {{realestateType}}, {{region}}, {{municipality}}",
-      TextPart: getNotificationEmailText(),
-      HtmlPart: getNotificationEmailHtml()
-    }
-  }
-
-  try {
-    const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise();
-    await templatePromise
-
-  } catch (e) {
-    console.log("Could not create MarketAlertEmailTemplate", e);
-  }
-}
-
-const getSubject = (realEstateType, region, municipality) => {
-  return `${getRealEstateTypeEnum(realEstateType).title} ${getRegionName(region)}, ${getMunicipalityName(region, municipality)}`
-}
-
 module.exports = {
-  sendTemplatedEmail,
+  sendTemplatedEmail
-  sendBulkEmail,
-  createMarketAlertEmailTemplate
 };

app/helpers/crawlers/olxClawler.js

@@ -1,6 +1,6 @@
 const fetch = require('node-fetch');
 const cheerio = require('cheerio');
-const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
+const { allRERequest, findPointInsideBoundingBox } = require('../url');
 const { getRealEstateTypeEnum } = require('../enums');
 const { getRegion, getMunicipality } = require('../codes')
 const Promise = require("bluebird");
@@ -13,283 +13,21 @@ module.exports = class OlxCrawler {
         this.maxResults = maxResults;
     }
 
-    async indexPages(urls) {
+    async indexSingle(url, email) {
-        const indexers = [];
-
-        urls.forEach(url => {
-            indexers.push(new Indexer(url));
-        });
-
-        return Promise.map(indexers, function (indexer) {
-            return indexer.indexWithPagination();
-        }).then(async (results) => {
-            return results
-        })
-    }
-
-    async crawl() {
-        console.log("OLX CRAWLER: start crawl");
-
-        const filteredResults = [];
-        const realestateRequests = await allRERequest();
-        console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
-        const urls = this.createRequestUrls(realestateRequests);
-        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
-        console.log("Final crawler results");
-        if (results[0]) {
-            console.log(results[0].length);
-
-            for (const finalResult of results[0]) {
-
-                if (null !== finalResult) {
-                    if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
-                        const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
-
-                    
-                        if (pointInsideBoundingBox[0].length !== 0) {
-                            finalResult.hasLocation = true
-                            filteredResults.push(finalResult);
-                        } else {
-                            finalResult.hasLocation = false
-                            filteredResults.push(finalResult);
-                        }
-                    }
-                }
-            }
-
-            console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
-            return filteredResults;
-        }
-        return []
-    }
-
-    createRequestUrls(realestateRequests) {
-        const urls = []
-
-        for (const request of realestateRequests) {
-            const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
-            const region = "kanton=" + getRegion(request.region).olxid;
-            const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
-            const sizeMin = "kvadrata_min=" + request.sizeMin;
-            const sizeMax = "kvadrata_max=" + request.sizeMax;
-            const priceMin = "od=" + request.priceMin;
-            const priceMax = "do=" + request.priceMax;
-
-            const olxUrl = {
-                url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`,
-                email: request.email,
-                uuid: request.uniqueId
-            }
-            console.log(olxUrl.url);
-            urls.push(olxUrl);
-        }
-
-        return urls;
-    }
-};
-
-
-class Indexer {
-
-    /**
-     * 
-     * @param {String|Array} olxUrl single or array of objects containing url email and uuid 
-     * @param {Array} hrefResutls array contaning urls from crawler results
-     */
-
-    constructor(olxUrl, hrefResutls) {
-        this.olxUrl = olxUrl;
-        this.hrefResutls = hrefResutls;
-    }
-
-    async indexWithPagination(pageNumber = 1) {
-
-        console.log("This is olxUrl:" + this.olxUrl.url);
-        const pageNr = this.olxUrl.url.match(/\d+$/);
-        const indexers = this.prepareIndexers(pageNumber ? [pageNumber] : pageNr);
-
         try {
-
+            const res = await fetch(url);
-            return Promise.map(indexers.indexers, function (indexer) {
-                return indexer.indexPage(pageNumber);
-            }).then(async (results) => {
-                let hasResults = false;
-
-                results.forEach(result => {
-                    if (!hasResults) {
-                        console.log("No results detected")
-                        hasResults = result.hasResults
-                    }
-                });
-
-                if (!hasResults) {
-                    console.log("HAS NO MORE RESULTS, stop the paging, there are some results and they should contain only HREFS");
-                    console.log(results.length);
-                    const singlePageIndexers = this.prepareHrefIndexers(results);
-                    if (singlePageIndexers.length === 0) {
-                        console.log("THERE IS NOT EVEN SINGLE RESULT");
-                        return []
-                    }
-
-                    return Promise.map(singlePageIndexers, function (indexer) {
-                        return indexer.indexSingle();
-                    }).then(async (results) => {
-                        console.log("SinglePageMethod in HAS NO RESULTS, MarketAralms");
-                        console.log(results.length);
-                        return results;
-                    });
-
-                } else {
-                    console.log("HAS MORE RESULTS, should only contain HREFS");
-                    console.log(results.length);
-                    const newResults = await this.indexWithPagination(results[0].pageNumber + 5);
-                    const singlePageIndexers = this.prepareHrefIndexers(results);
-
-                    const newerResults = await Promise.map(singlePageIndexers, function (indexer) {
-                        return indexer.indexSingle();
-                    }).then(async (results) => {
-                        console.log("SinglePageMethod HAS RESULTS, should contain MarketAlerts only");
-                        console.log(results.length);
-                        return results;
-                    });
-
-                    Array.prototype.push.apply(newResults, newerResults);
-                    return newResults;
-
-                }
-            });
-        } catch (e) {
-            console.error("Error has accured", e);
-        }
-
-    }
-
-    prepareIndexers(pageNr) {
-        console.log("Entering prepareIndexers : page nr - " + pageNr);
-        const indexers = [];
-        let lastPageNumber;
-        if (pageNr) {
-            for (let index = Number(pageNr[0]); index <= Number(pageNr[0]) + 5; index++) {
-                lastPageNumber = index;
-                const newOlxUrl = {
-                    url: this.olxUrl.url.replace(/\d+$/, "") + index,
-                    email: this.olxUrl.email,
-                    uuid: this.olxUrl.uuid
-                }
-                indexers.push(new Indexer(newOlxUrl));
-
-            }
-        } else {
-            for (let index = 1; index <= 5; index++) {
-                lastPageNumber = index;
-                const newOlxUrl = {
-                    url: this.olxUrl.url + index,
-                    email: this.olxUrl.email,
-                    uuid: this.olxUrl.uuid
-                }
-                indexers.push(new Indexer(newOlxUrl));
-            }
-        }
-        return {
-            indexers: indexers,
-            lastPageNumber: lastPageNumber
-        };
-    }
-
-    prepareHrefIndexers(results) {
-        const indexers = []
-
-        if (!Array.isArray(results)) {
-            results.hrefs.forEach(href => {
-                const newOlxUrl = {
-                    url: href,
-                    email: results.olxUrl.email,
-                    uuid: results.olxUrl.uuid
-                }
-
-                indexers.push(new Indexer(newOlxUrl));
-            });
-
-        } else {
-
-
-            results.forEach(result => {
-
-                if (result !== null && result.hasOwnProperty('hrefs')) {
-                    result.hrefs.forEach(href => {
-                        // console.log(href);
-                        const newOlxUrl = {
-                            url: href,
-                            email: result.olxUrl.email,
-                            uuid: result.olxUrl.uuid
-                        }
-
-                        indexers.push(new Indexer(newOlxUrl));
-                    })
-                }
-
-            });
-        }
-
-        return indexers;
-    }
-
-    async indexPage(pageNumber) {
-        console.log("Page number in index page, max page number :")
-        console.log(pageNumber);
-
-        try {
-
-            console.log("Indexing page: " + this.olxUrl.url);
-            const res = await fetch(this.olxUrl.url);
-            const body = await res.text();
-            const $ = cheerio.load(body);
-            const hrefs = [];
-            let hasResults = false
-
-            $('#rezultatipretrage').find('.listitem').each((i, elem) => {
-                hasResults = true
-                const href = $(elem).find('a').first().attr('href');
-                hrefs.push(href);
-            });
-
-            console.log("this is hrefs for olxUrl" + this.olxUrl.url);
-            console.log("NUMBER OF HREFS " + hrefs.length);
-
-            return {
-                hrefs: hrefs,
-                hasResults: hasResults,
-                pageNumber: pageNumber,
-                olxUrl: this.olxUrl
-            }
-        } catch (e) {
-            console.error('Exception caught:' + e);
-        }
-    }
-
-    async indexSingle() {
-        try {
-            console.log("Index single");
-            console.log(this.olxUrl.url);
-
-            if (this.olxUrl.url === undefined) {
-                return {}
-            }
-
-            if (global.hrefs) {
-
-                if (global.hrefs[this.olxUrl.uuid] && global.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
-
-                    console.log("We found duplicate URL");
-                    return null
-                }
-            }
-
-            const res = await fetch(this.olxUrl.url);
             const body = await res.text();
             const $ = cheerio.load(body);
 
-            const title = $('#naslovartikla').text().trim();
+            //TODO figure out what to do with username
+            const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+            // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+            //     return null;
+            // }
+
+            //TODO remove properties that are not needed, and add some if they are missing
+            const title = $('#naslovartikla').text();
             const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
 
             const price = $('#pc > p:nth-child(2)').text();
@@ -299,17 +37,43 @@ class Indexer {
             const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
             const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
 
+            const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
             const time = $('time').attr('datetime');
             const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
 
             const descriptions = $('.artikal_detaljniopis_tekst');
+            // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
             const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
             const imgRe = /href":("[^"]*")/g;
             const matches = latLngRe.exec(body);
             let lng = '',
                 lat = '';
+
+
+            const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
             const parsePrice = (price) => parseFloat(price.replace(".", ""))
 
+
+            // TODO we dont save images ??
+
+            // const images = [];
+            // const imgMatches = body.match(imgRe);
+
+            // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+            //     let img = imgMatches[i].replace("href\":", "")
+            //     img = img.replace("\"", "");
+            //     img = img.replace("\"", "");
+            //     images.push(img);
+            // }
+
+            // const uploadPromises = images.map(img => {
+            //     const imgFixed = eval(`'${img}'`);
+            //     return cloudinary.uploader.upload(eval(`'${img}'`));
+            // });
+
+            // const uploadResults = await Promise.all(uploadPromises);
+            // const cloudinaryImages = uploadResults.map(ur => ur.url);
+
             if (matches && matches.length >= 3) {
                 lat = matches[1];
                 lng = matches[2];
@@ -323,23 +87,25 @@ class Indexer {
 
             const data = {
                 realEstateType: this.getCategoryId(realEstateType),
-                email: this.olxUrl.email,
+                email : email,
-                uuid: this.olxUrl.uuid,
                 olxId: olxId,
-                url: this.olxUrl.url,
+                // category: category,
+                url,
                 title,
-                price: isNaN(parsedPrice) ? 0 : parsedPrice,
+                price: isNaN(parsedPrice) ? price : parsedPrice,
                 size: parseFloat(size),
-                gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
+                gardenSize: parseFloat(gardenSize),
                 address,
                 region,
                 municipality,
+                // adType: AD_TYPE_SALE,
                 time,
                 shortDescription: descriptions.first().text(),
                 longDescription: descriptions.last().text(),
                 lat,
                 lng,
                 loc: [parseFloat(lat), parseFloat(lng)],
+                // images: cloudinaryImages
             };
 
             return data;
@@ -350,21 +116,292 @@ class Indexer {
         return null;
     }
 
-    getCategoryId(category) {
+    async indexPage(olxUrl, maxResults = 1000) {
+        try {
+            // console.log('Starting to index page: ' + pageNr);
+            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
 
-        switch (category) {
+            const res = await fetch(olxUrl.url);
-            case 'Stanovi':
+            const body = await res.text();
-                return 'stan';
+            const $ = cheerio.load(body);
+            const hrefs = [];
+            const results = [];
 
-            case 'Vikendice':
+            $('#rezultatipretrage').find('.listitem').each((i, elem) => {
-                return 'vikendica'
+                const href = $(elem).find('a').first().attr('href');
+                hrefs.push(href);
+            });
 
-            case 'Kuće':
+            let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
-                return 'kuca';
 
-            default:
+            for (let i = 0; i < hrefs.length; i++) {
-                return '';
+                console.log(`indexing: ${hrefs[i]}`);
+
+                const singleData = await this.indexSingle(hrefs[i], olxUrl.email);
+
+                if (singleData) {
+                    results.push(singleData);
+                }
+                // await this.sleep(500);
+            }
+
+            return results;
+        } catch (e) {
+            console.error('Exception caught:' + e);
         }
     }
-}
 
+    getCategoryId (category) {
+        if (category === 'Stanovi') {
+          return 'stan';
+        } else if (category === 'Vikendice') {
+          return 'vikendica';
+        } else if (category === 'Kuće') {
+          return 'kuca';
+        }
+      }
+
+    async indexPages(urls, start, end, maxResults = 1000) {
+        // let results = {};
+        // for (let i = start; i <= end; i++) {
+        //     let result = await this.indexPage(i, maxResults);
+        //     Object.assign(results, result)
+        //     await this.sleep(5000);
+        // }
+        // return results;
+
+        let results = [];
+        const indexers= [];
+        let it = 3
+        for (let url of urls) {
+            // let result = await this.indexPage(url, maxResults);
+            // results.push(result);
+            it++
+            indexers.push(new Indexer(it * 2000));
+
+        }
+        return Promise.map(indexers, function (indexer) {
+            return indexer.indexPage();
+        }).then(async (results) => {
+            return results
+        })       
+    }
+
+    async crawl() {
+
+        const filteredResults = [];
+        const realestateRequests = await allRERequest()
+        const urls = this.createRequestUrls(realestateRequests);
+        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
+
+        for (const result of results) {
+            for (const re1 of result) {
+                if (re1.lat !== undefined && re1.lat !== null && re1.lat !== "") {
+                    const pointInsideBoundingBox = await findPointInsideBoundingBox([re1.lng, re1.lat]);
+
+                    if (pointInsideBoundingBox[0].length !== 0) {
+                        filteredResults.push(re1);
+                    }
+                }
+            }
+        }
+        // await this.sleep(10000);
+        console.log(filteredResults);
+        return filteredResults;
+    }
+
+    async sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+      }
+
+    createRequestUrls(realestateRequests) {
+        const urls = []
+
+        for (const request of realestateRequests) {
+            const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
+            const region = "kanton=" + getRegion(request.region).olxid;
+            const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
+            const sizeMin = "kvadrata_min=" + request.sizeMin;
+            const sizeMax = "kvadrata_max=" + request.sizeMax;
+            const priceMin = "od=" + request.priceMin;
+            const priceMax = "do=" + request.priceMax;
+
+            const olxUrl = {
+                url: "https://www.olx.ba/pretraga?" + realsestateType + "&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&" + region + "&" + municipality + "&" + priceMin + "&" + priceMax + "&vrsta=samoprodaja&" + sizeMin + "&" + sizeMax,
+                email: request.email
+            }
+            console.log(olxUrl.url);
+            urls.push(olxUrl);
+        }
+
+        return urls;
+    }
+};
+
+
+class Indexer {
+    
+    constructor(olxUrl, email, uuid) {
+        this.olxUrl = olxUrl
+        this.email = email
+        this.uuid = uuid
+    }
+
+    async indexPage() {
+
+        try {
+            // console.log('Starting to index page: ' + pageNr);
+            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+
+            // const res = await fetch(this.olxUrl.url);
+            // const body = await res.text();
+            // const $ = cheerio.load(body);
+            // const hrefs = [];
+            // const results = [];
+        
+            // $('#rezultatipretrage').find('.listitem').each((i, elem) => {
+            //     const href = $(elem).find('a').first().attr('href');
+            //     hrefs.push(href);
+            // });
+
+            // let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
+
+            // for (let i = 0; i < hrefs.length; i++) {
+            //     console.log(`indexing: ${hrefs[i]}`);
+
+            //     const singleData = await this.indexSingle(hrefs[i], this.olxUrl.email);
+
+            //     if (singleData) {
+            //         results.push(singleData);
+            //     }
+            //     // await this.sleep(500);
+            // }
+            await this.sleep(this.olxUrl);
+            console.log('Finished indexing PAGE');
+            const singleIndex = [new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl)]
+            return Promise.map(singleIndex, function (indexer) {
+                return indexer.indexSingle();
+            }).then(async (results) => {
+               
+                return results
+            })  
+
+
+          
+            // return results;
+        } catch (e) {
+            console.error('Exception caught:' + e);
+        }
+    }
+
+    async indexSingle() {
+        // try {
+        //     const res = await fetch(url);
+        //     const body = await res.text();
+        //     const $ = cheerio.load(body);
+
+        //     //TODO figure out what to do with username
+        //     const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+        //     // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+        //     //     return null;
+        //     // }
+
+        //     //TODO remove properties that are not needed, and add some if they are missing
+        //     const title = $('#naslovartikla').text();
+        //     const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
+
+        //     const price = $('#pc > p:nth-child(2)').text();
+        //     const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
+        //     const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
+        //     const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
+        //     const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
+        //     const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+        //     const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+        //     const time = $('time').attr('datetime');
+        //     const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+        //     const descriptions = $('.artikal_detaljniopis_tekst');
+        //     // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
+        //     const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+        //     const imgRe = /href":("[^"]*")/g;
+        //     const matches = latLngRe.exec(body);
+        //     let lng = '',
+        //         lat = '';
+
+
+        //     const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+        //     const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
+        //     // TODO we dont save images ??
+
+        //     // const images = [];
+        //     // const imgMatches = body.match(imgRe);
+
+        //     // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+        //     //     let img = imgMatches[i].replace("href\":", "")
+        //     //     img = img.replace("\"", "");
+        //     //     img = img.replace("\"", "");
+        //     //     images.push(img);
+        //     // }
+
+        //     // const uploadPromises = images.map(img => {
+        //     //     const imgFixed = eval(`'${img}'`);
+        //     //     return cloudinary.uploader.upload(eval(`'${img}'`));
+        //     // });
+
+        //     // const uploadResults = await Promise.all(uploadPromises);
+        //     // const cloudinaryImages = uploadResults.map(ur => ur.url);
+
+        //     if (matches && matches.length >= 3) {
+        //         lat = matches[1];
+        //         lng = matches[2];
+        //     }
+
+        //     const parsedPrice = parsePrice(price);
+
+        //     const locationArray = location.split(",");
+        //     const region = locationArray[0];
+        //     const municipality = locationArray[1];
+
+        //     const data = {
+        //         realEstateType: this.getCategoryId(realEstateType),
+        //         email : email,
+        //         olxId: olxId,
+        //         // category: category,
+        //         url,
+        //         title,
+        //         price: isNaN(parsedPrice) ? price : parsedPrice,
+        //         size: parseFloat(size),
+        //         gardenSize: parseFloat(gardenSize),
+        //         address,
+        //         region,
+        //         municipality,
+        //         // adType: AD_TYPE_SALE,
+        //         time,
+        //         shortDescription: descriptions.first().text(),
+        //         longDescription: descriptions.last().text(),
+        //         lat,
+        //         lng,
+        //         loc: [parseFloat(lat), parseFloat(lng)],
+        //         // images: cloudinaryImages
+        //     };
+
+        //     return data;
+        // } catch (e) {
+        //     console.error('Exception caught: ' + e.message);
+        // }
+
+        // return null;
+        await this.sleep(this.olxUrl);
+        console.log("Finished indexing single page");
+        return {};
+    }
+
+    async sleep(ms) {
+        console.log(ms);
+        return new Promise(resolve => setTimeout(resolve, ms));
+      }
+}

app/helpers/db/dbHelper.js

@@ -1,70 +1,10 @@
-const db = require('../../models/index');
 
-/**
+// const db = require('../../models/index');
- * Find all subscribed RealEstateRequests 
- */
-const allRERequest = async () => {
-    return await db.RealEstateRequest.findAll({
-        where: {
-            subscribed: true
-        }
-      });
-}
 
-/**
- * Find all subscribed RealEstateRequests by UUID
- */
-const allRERequestByUiid = async (requestArray) => {
 
-  const Op = db.Sequelize.Op;
+// const bulkInsert = async (reuslts) => {
-  return await db.RealEstateRequest.findAll({
+//     db.MarketAlert.bulkCreate({
-      where: {
-          subscribed: true,
-          [Op.or]: requestArray
-      }
-    });
-}
 
-/**
+//     })
- * Find all , or all depending on notified bolean marketalerts, that the hasLocation is true, and order them by email
+   
- * 
+//   }
- * @param fechAll bolean
- * @param notified bolean
- * 
- * @returns array of MarketAlerts
- */
-const allMarketAlerts = async (fetchAll, notified) => {
-
-    let queryObject = {
-        order: [
-            ['email', 'DESC'],
-        ]
-    }
-
-    if (!fetchAll){
-        queryObject.where = {
-            notified: notified,
-            hasLocation: true
-        }
-    }
-
-    return await db.MarketAlert.findAll(queryObject);
-  }
-  
-/**
- * Find all unnotified marketalerts 
- *  @param latLng array
- *  @param email string
- * 
- * @returns array of MarketAlerts
- */
-const findPointInsideBoundingBox = async (latLng, email) => {
-  return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
-}
-
-module.exports = {
-  allRERequest,
-  allMarketAlerts,
-  allRERequestByUiid,
-  findPointInsideBoundingBox
-};

app/helpers/url.js

@@ -7,6 +7,17 @@ const currentRERequest = async (req) => {
   const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
   return request;
 };
+// TODO Fetch only subscribed realestate requests
+const allRERequest = async () => {
+  return await db.RealEstateRequest.findAll();
+}
+
+const findPointInsideBoundingBox = async (latLng) => {
+  return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
+}
+
 module.exports = {
   currentRERequest,
+  allRERequest,
+  findPointInsideBoundingBox
 };

app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js

@@ -1,20 +0,0 @@
-'use strict';
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'notified',
-        {
-          type: Sequelize.BOOLEAN
-        }
-    );
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'notified'
-    );
-  }
-};

app/migrations/20190628165512-add-title-to-marketalerts.js

@@ -1,20 +0,0 @@
-'use strict';
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'title',
-        {
-          type: Sequelize.STRING
-        }
-    );
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'title'
-    );
-  }
-};

app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js

@@ -1,20 +0,0 @@
-'use strict';
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'request',
-        {
-          type: Sequelize.STRING
-        }
-    );
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'request'
-    );
-  }
-};

app/migrations/20190710141356-add-has-location-to-marketalerts.js

@@ -1,20 +0,0 @@
-'use strict';
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'hasLocation',
-        {
-          type: Sequelize.BOOLEAN
-        }
-    );
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'hasLocation'
-    );
-  }
-};

app/models/marketalert.js

@@ -11,10 +11,6 @@ module.exports = (sequelize, DataTypes) => {
     municipality : DataTypes.STRING,
     region : DataTypes.STRING,
     realEstateType : DataTypes.STRING,
-    notified : DataTypes.BOOLEAN,
-    title : DataTypes.STRING,
-    request: DataTypes.STRING,
-    hasLocation: DataTypes.BOOLEAN,
     
     email: { 
       type: DataTypes.STRING,

app/services/crawlerService.js

@@ -2,46 +2,26 @@
 const Promise = require("bluebird");
 const OlxCrawler = require("../helpers/crawlers/olxClawler");
 const db = require("../models/index");
-const { allMarketAlerts } = require('../helpers/db/dbHelper');
 
 const olxCrawler = new OlxCrawler(1, 2, 3);
+const olxCrawler1 = new OlxCrawler(1, 2, 3);
+const olxCrawler2 = new OlxCrawler(1, 2, 3);
 
 const crawlers = [
     olxCrawler,
+    olxCrawler1,
+    olxCrawler2,
 ];
 
 async function crawlAll() {
-    console.log("CRAWLER SERVICE: crawlAll");
 
-    try {
+    Promise.map(crawlers, function (crawler) {
-        const marketAlertsFromDb = await allMarketAlerts(true);
-        const hrefs = [];
-       
-        marketAlertsFromDb.map(marketAlert => {
-            if (hrefs[marketAlert.request] === undefined) {
-                hrefs[marketAlert.request] = []
-            }
-
-            hrefs[marketAlert.request].push(marketAlert.url);
-        })
-
-        global.hrefs = hrefs;
-        console.log("CRAWLER SERVICE: GLOBAL HREFS");
-        console.log(global.hrefs);
-
-    } catch (e) {
-        console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
-    }
-
-    return Promise.map(crawlers, function (crawler) {
         return crawler.crawl();
     }).then(async (results) => {
 
         try {
 
-            const marketAlertsFromDb = await allMarketAlerts(false, true);
+            const marketAlertsFromDb = await db.MarketAlert.findAll();
-
-            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
 
             const marketAlerts = [];
             const mergedResults = [].concat.apply([], results);
@@ -54,33 +34,25 @@ async function crawlAll() {
                     size: result.size,
                     price: result.price,
                     email: result.email,
-                    request: result.uuid,
                     // lastDate: DataTypes.STRING,
                     municipality: result.municipality,
                     region: result.region,
                     gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
-                    realEstateType: result.realEstateType,
+                    realEstateType: result.realEstateType
-                    title: result.title,
-                    notified: false,
-                    hasLocation: result.hasLocation
                 })
             }
-            console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
-
             try {
-
+                const filteredMarketAlerts =  marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
-                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
-                console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
-
                 await db.MarketAlert.bulkCreate(filteredMarketAlerts);
-
+                process.exit()
             } catch (e) {
-                console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
+                console.log("Could not bulkCreate marketalers reason: ", e);
             }
         } catch (e) {
-            console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
+            console.log("Error crawling. Trying next crawler! ", e);
         }
     })
 };
-module.exports = crawlAll;
+
-// crawlAll();
+crawlAll();
+

app/services/notificationService.js

@@ -1,29 +0,0 @@
-
-const db = require("../models/index");
-const { allMarketAlerts } = require('../helpers/db/dbHelper');
-const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/awsEmail');
-
-
-async function processNotifications() {
-
-    try {
-        const marketAlerts = await allMarketAlerts(false, false);
-        console.log(marketAlerts.length)
-        await createMarketAlertEmailTemplate();
-        if (marketAlerts.length > 0) {
-            console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length)
-            await sendBulkEmail(marketAlerts);
-        } else {
-            console.log("NOTIFICATION SERVICE: No new alerts");
-        }
-
-        await db.MarketAlert.update(
-            { notified: true }, /* set attributes' value */
-            { where: { notified: false } } /* where criteria */
-        );
-    } catch (e) {
-        console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
-    }
-}
-
-module.exports = processNotifications;

index.js

@@ -10,9 +10,6 @@ const { getQuerySubmit, postQuerySubmit } = require('./app/controllers/querySubm
 const { getGoAgain } = require('./app/controllers/goAgain');
 const { getNeighborhood, postNeighborhood } = require('./app/controllers/neighborhoodMap');
 const { getUnsubscribe } = require('./app/controllers/unsubscribe');
-const schedule = require('node-schedule');
-const crawlAll = require('./app/services/crawlerService')
-const processNotifications = require('./app/services/notificationService')
 
 let express = require("express");
 const path = require("path");
@@ -119,25 +116,6 @@ app.post("/api/payforalert", (req, res) => {
   });
 });
 
-var runServices = async () => {
-
-  
-}
-
-runServices();
-
-var rule = new schedule.RecurrenceRule();
-  rule.seccond = 1;
-  schedule.scheduleJob(rule, async function () {
-    console.log(new Date(), 'Crawler service started');
-    await crawlAll();
-    console.log(new Date(), 'Crawler service finished, starting Notification service');
-    await processNotifications();
-    console.log(new Date(), 'Notification service finished');
-  });
-
-
-
 app.get('/', welcome);
 app.get('/vrstanekretnine/:request_id', getRealEstateTypes);
 app.get('/vrstanekretnine', getRealEstateTypes);

package-lock.json

@@ -559,38 +559,13 @@
       }
     },
     "cliui": {
-      "version": "5.0.0",
+      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
-      "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==",
+      "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
       "requires": {
-        "string-width": "^3.1.0",
+        "string-width": "^2.1.1",
-        "strip-ansi": "^5.2.0",
+        "strip-ansi": "^4.0.0",
-        "wrap-ansi": "^5.1.0"
+        "wrap-ansi": "^2.0.0"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
-          "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg=="
-        },
-        "string-width": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
-          "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
-          "requires": {
-            "emoji-regex": "^7.0.1",
-            "is-fullwidth-code-point": "^2.0.0",
-            "strip-ansi": "^5.1.0"
-          }
-        },
-        "strip-ansi": {
-          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
-          "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
-          "requires": {
-            "ansi-regex": "^4.1.0"
-          }
-        }
       }
     },
     "cls-bluebird": {
@@ -602,6 +577,11 @@
         "shimmer": "^1.1.0"
       }
     },
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
+    },
     "collection-visit": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
@@ -721,9 +701,9 @@
       "dev": true
     },
     "core-js": {
-      "version": "2.6.9",
+      "version": "2.6.5",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.5.tgz",
-      "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A=="
+      "integrity": "sha512-klh/kDpwX8hryYL14M9w/xei6vrv6sE8gTHDG7/T/+SEovB/G4ejwcfE/CBzO6Edsu+OETZMZ3wcX/EjUkrl5A=="
     },
     "core-util-is": {
       "version": "1.0.2",
@@ -739,15 +719,6 @@
         "capture-stack-trace": "^1.0.0"
       }
     },
-    "cron-parser": {
-      "version": "2.12.0",
-      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.12.0.tgz",
-      "integrity": "sha512-1GU6CQJ6gT9XDEGeTuzfhZgFMf82BSs3ihFA3i2wr4qGKJLhO1kOvaIF9biIo39CaPgzZ17U8FgYxRv/+UR50A==",
-      "requires": {
-        "is-nan": "^1.2.1",
-        "moment-timezone": "^0.5.25"
-      }
-    },
     "cross-spawn": {
       "version": "6.0.5",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
@@ -783,12 +754,11 @@
       "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
     },
     "d": {
-      "version": "1.0.1",
+      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/d/-/d-1.0.0.tgz",
-      "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
+      "integrity": "sha1-dUu1v+VUUdpppYuU1F9MWwRi1Y8=",
       "requires": {
-        "es5-ext": "^0.10.50",
+        "es5-ext": "^0.10.9"
-        "type": "^1.0.1"
       }
     },
     "dashdash": {
@@ -829,14 +799,6 @@
       "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.2.1.tgz",
       "integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA=="
     },
-    "define-properties": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
-      "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
-      "requires": {
-        "object-keys": "^1.0.12"
-      }
-    },
     "define-property": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
@@ -979,11 +941,6 @@
       "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.6.1.tgz",
       "integrity": "sha512-0xy4A/twfrRCnkhfk8ErDi5DqdAsAqeGxht4xkCUrsvhhbQNs7E+4jV0CN7+NKIY0aHE72+XvqtBIXzD31ZbXQ=="
     },
-    "emoji-regex": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
-      "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA=="
-    },
     "encodeurl": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
@@ -1003,9 +960,9 @@
       "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
     },
     "es5-ext": {
-      "version": "0.10.50",
+      "version": "0.10.49",
-      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.50.tgz",
+      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.49.tgz",
-      "integrity": "sha512-KMzZTPBkeQV/JcSQhI5/z6d9VWJ3EnQ194USTUwIYZ2ZbpN8+SGXQKt1h68EX44+qt+Fzr8DO17vnxrw7c3agw==",
+      "integrity": "sha512-3NMEhi57E31qdzmYp2jwRArIUsj1HI/RxbQ4bgnSB+AIKIxsAmTiK83bYMifIcpWvEc3P1X30DhUKOqEtF/kvg==",
       "requires": {
         "es6-iterator": "~2.0.3",
         "es6-symbol": "~3.1.1",
@@ -1032,13 +989,13 @@
       }
     },
     "es6-weak-map": {
-      "version": "2.0.3",
+      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.2.tgz",
-      "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==",
+      "integrity": "sha1-XjqzIlH/0VOKH45f+hNXdy+S2W8=",
       "requires": {
         "d": "1",
-        "es5-ext": "^0.10.46",
+        "es5-ext": "^0.10.14",
-        "es6-iterator": "^2.0.3",
+        "es6-iterator": "^2.0.1",
         "es6-symbol": "^3.1.1"
       }
     },
@@ -1396,8 +1353,7 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -1418,14 +1374,12 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -1440,20 +1394,17 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -1570,8 +1521,7 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -1583,7 +1533,6 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -1598,7 +1547,6 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -1606,14 +1554,12 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "minipass": {
           "version": "2.3.5",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -1632,7 +1578,6 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -1713,8 +1658,7 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -1726,7 +1670,6 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -1812,8 +1755,7 @@
         "safe-buffer": {
           "version": "5.1.2",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -1849,7 +1791,6 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -1869,7 +1810,6 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -1913,14 +1853,12 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "yallist": {
           "version": "3.0.3",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         }
       }
     },
@@ -1930,9 +1868,9 @@
       "integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w=="
     },
     "get-caller-file": {
-      "version": "2.0.5",
+      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
-      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
+      "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w=="
     },
     "get-stream": {
       "version": "4.1.0",
@@ -1957,9 +1895,9 @@
       }
     },
     "glob": {
-      "version": "7.1.4",
+      "version": "7.1.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
-      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+      "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
       "requires": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -2305,14 +2243,6 @@
         "is-path-inside": "^1.0.0"
       }
     },
-    "is-nan": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.2.1.tgz",
-      "integrity": "sha1-n69ltvttskt/XAYoR16nH5iEAeI=",
-      "requires": {
-        "define-properties": "^1.1.1"
-      }
-    },
     "is-npm": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz",
@@ -2423,14 +2353,14 @@
       "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
     },
     "js-beautify": {
-      "version": "1.10.0",
+      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.10.0.tgz",
+      "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.9.1.tgz",
-      "integrity": "sha512-OMwf/tPDpE/BLlYKqZOhqWsd3/z2N3KOlyn1wsCRGFwViE8LOQTcDtathQvHvZc+q+zWmcNAbwKSC+iJoMaH2Q==",
+      "integrity": "sha512-oxxvVZdOdUfzk8IOLBF2XUZvl2GoBEfA+b0of4u2EBY/46NlXasi8JdFvazA5lCrf9/lQhTjyVy2QCUW7iq0MQ==",
       "requires": {
         "config-chain": "^1.1.12",
-        "editorconfig": "^0.15.3",
+        "editorconfig": "^0.15.2",
         "glob": "^7.1.3",
-        "mkdirp": "~0.5.1",
+        "mkdirp": "~0.5.0",
         "nopt": "~4.0.1"
       }
     },
@@ -2510,11 +2440,6 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
       "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
     },
-    "long-timeout": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz",
-      "integrity": "sha1-lyHXiLR+C8taJMLivuGg2lXatRQ="
-    },
     "lowercase-keys": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
@@ -2760,16 +2685,6 @@
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz",
       "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA=="
     },
-    "node-schedule": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-1.3.2.tgz",
-      "integrity": "sha512-GIND2pHMHiReSZSvS6dpZcDH7pGPGFfWBIEud6S00Q8zEIzAs9ommdyRK1ZbQt8y1LyZsJYZgPnyi7gpU2lcdw==",
-      "requires": {
-        "cron-parser": "^2.7.3",
-        "long-timeout": "0.1.1",
-        "sorted-array-functions": "^1.0.0"
-      }
-    },
     "nodemon": {
       "version": "1.19.0",
       "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.0.tgz",
@@ -2836,6 +2751,11 @@
         "boolbase": "~1.0.0"
       }
     },
+    "number-is-nan": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
+      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
+    },
     "oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -2877,11 +2797,6 @@
         }
       }
     },
-    "object-keys": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
-      "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
-    },
     "object-visit": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
@@ -3394,14 +3309,14 @@
       "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
     },
     "require-main-filename": {
-      "version": "2.0.0",
+      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz",
-      "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg=="
+      "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE="
     },
     "resolve": {
-      "version": "1.11.1",
+      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz",
-      "integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==",
+      "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==",
       "requires": {
         "path-parse": "^1.0.6"
       }
@@ -3525,9 +3440,9 @@
       }
     },
     "sequelize-cli": {
-      "version": "5.5.0",
+      "version": "5.4.0",
-      "resolved": "https://registry.npmjs.org/sequelize-cli/-/sequelize-cli-5.5.0.tgz",
+      "resolved": "https://registry.npmjs.org/sequelize-cli/-/sequelize-cli-5.4.0.tgz",
-      "integrity": "sha512-twVQ02alCpr2XvxNmpi32C48WZs6xHTH1OFTfTS5Meg3BVqOM8ghiZoml4FITFjlD8sAJSQjlAHTwqTbuolA6Q==",
+      "integrity": "sha512-4Gvl0yH0T3hhSdiiOci3+IKIfVG9x2os0hGWsbfa8QuyGgk9mZOqgTBnSCRtuxsdAyzUix9kfcTnfNolVNtprg==",
       "requires": {
         "bluebird": "^3.5.3",
         "cli-color": "^1.4.0",
@@ -3536,7 +3451,7 @@
         "lodash": "^4.17.5",
         "resolve": "^1.5.0",
         "umzug": "^2.1.0",
-        "yargs": "^13.1.0"
+        "yargs": "^12.0.5"
       }
     },
     "serve-static": {
@@ -3718,11 +3633,6 @@
         }
       }
     },
-    "sorted-array-functions": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.2.0.tgz",
-      "integrity": "sha512-sWpjPhIZJtqO77GN+LD8dDsDKcWZ9GCOJNqKzi1tvtjGIzwfoyuRH8S0psunmc6Z5P+qfDqztSbwYR5X/e1UTg=="
-    },
     "source-map": {
       "version": "0.5.7",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
@@ -3811,7 +3721,6 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
       "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
-      "dev": true,
       "requires": {
         "is-fullwidth-code-point": "^2.0.0",
         "strip-ansi": "^4.0.0"
@@ -3829,7 +3738,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
       "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
-      "dev": true,
       "requires": {
         "ansi-regex": "^3.0.0"
       },
@@ -3837,8 +3745,7 @@
         "ansi-regex": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="
-          "dev": true
         }
       }
     },
@@ -4037,11 +3944,6 @@
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
       "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
     },
-    "type": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/type/-/type-1.0.1.tgz",
-      "integrity": "sha512-MAM5dBMJCJNKs9E7JXo4CXRAansRfG0nlJxW7Wf6GZzSOvH31zClSaHdIMWLehe/EGMBkqeC55rrkaOr5Oo7Nw=="
-    },
     "type-is": {
       "version": "1.6.16",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
@@ -4304,36 +4206,38 @@
       }
     },
     "wrap-ansi": {
-      "version": "5.1.0",
+      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
-      "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==",
+      "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=",
       "requires": {
-        "ansi-styles": "^3.2.0",
+        "string-width": "^1.0.1",
-        "string-width": "^3.0.0",
+        "strip-ansi": "^3.0.1"
-        "strip-ansi": "^5.0.0"
       },
       "dependencies": {
-        "ansi-regex": {
+        "is-fullwidth-code-point": {
-          "version": "4.1.0",
+          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg=="
+          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+          "requires": {
+            "number-is-nan": "^1.0.0"
+          }
         },
         "string-width": {
-          "version": "3.1.0",
+          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-          "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
+          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
           "requires": {
-            "emoji-regex": "^7.0.1",
+            "code-point-at": "^1.0.0",
-            "is-fullwidth-code-point": "^2.0.0",
+            "is-fullwidth-code-point": "^1.0.0",
-            "strip-ansi": "^5.1.0"
+            "strip-ansi": "^3.0.0"
           }
         },
         "strip-ansi": {
-          "version": "5.2.0",
+          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
-          "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
+          "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
           "requires": {
-            "ansi-regex": "^4.1.0"
+            "ansi-regex": "^2.0.0"
           }
         }
       }
@@ -4390,52 +4294,28 @@
       "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
     },
     "yargs": {
-      "version": "13.2.4",
+      "version": "12.0.5",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.4.tgz",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz",
-      "integrity": "sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg==",
+      "integrity": "sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==",
       "requires": {
-        "cliui": "^5.0.0",
+        "cliui": "^4.0.0",
+        "decamelize": "^1.2.0",
         "find-up": "^3.0.0",
-        "get-caller-file": "^2.0.1",
+        "get-caller-file": "^1.0.1",
-        "os-locale": "^3.1.0",
+        "os-locale": "^3.0.0",
         "require-directory": "^2.1.1",
-        "require-main-filename": "^2.0.0",
+        "require-main-filename": "^1.0.1",
         "set-blocking": "^2.0.0",
-        "string-width": "^3.0.0",
+        "string-width": "^2.0.0",
         "which-module": "^2.0.0",
-        "y18n": "^4.0.0",
+        "y18n": "^3.2.1 || ^4.0.0",
-        "yargs-parser": "^13.1.0"
+        "yargs-parser": "^11.1.1"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
-          "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg=="
-        },
-        "string-width": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
-          "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
-          "requires": {
-            "emoji-regex": "^7.0.1",
-            "is-fullwidth-code-point": "^2.0.0",
-            "strip-ansi": "^5.1.0"
-          }
-        },
-        "strip-ansi": {
-          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
-          "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
-          "requires": {
-            "ansi-regex": "^4.1.0"
-          }
-        }
       }
     },
     "yargs-parser": {
-      "version": "13.1.1",
+      "version": "11.1.1",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-11.1.1.tgz",
-      "integrity": "sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ==",
+      "integrity": "sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==",
       "requires": {
         "camelcase": "^5.0.0",
         "decamelize": "^1.2.0"

package.json

@@ -7,6 +7,7 @@
     "test": "echo \"Error: no test specified\" && exit 1",
     "start": "node ./index.js",
     "start-mon": "nodemon ./index.js",
+    "scheduler": "node ./app/services/crawlerService.js",
     "migrate": "cd app && npx sequelize db:migrate",
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
@@ -34,11 +35,10 @@
     "express-ejs-layouts": "^2.5.0",
     "express-layout": "^0.1.0",
     "node-fetch": "^2.3.0",
-    "node-schedule": "^1.3.2",
     "pg": "^7.10.0",
     "react-step-wizard": "^5.1.0",
     "sequelize": "^4.43.2",
-    "sequelize-cli": "^5.5.0"
+    "sequelize-cli": "^5.4.0"
   },
   "devDependencies": {
     "nodemon": "^1.19.0"