Merge branch 'start-crawler-loop-with-server' into 'master'

start crawler loop when server is started

See merge request saburly/marketalarm/web!34

app/config/appConfig.js

@@ -8,8 +8,13 @@ const APP_URL =
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
+const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
+const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
+
 module.exports = {
   APP_PORT,
   APP_URL,
-  DEFAULT_TIMEZONE
+  DEFAULT_TIMEZONE,
+  CRAWLER_INTERVAL,
+  STOP_CRAWLER
 };

app/crawler/crawl.js

@@ -26,15 +26,14 @@ const crawlers = [
 async function crawlAll() {
   for (let crawler of crawlers) {
     try {
-      const newRealEstates = await crawler.crawl();
+      return await crawler.crawl();
-
-      console.log("Number of new real estates : ", newRealEstates.length);
     } catch (e) {
       console.log("Error crawling. Trying next crawler! ", e);
+      return [];
     }
   }
 }
 
-(async () => {
+module.exports = {
-  await crawlAll();
+  crawlAll
-})();
+};

app/crawler/npmCrawl.js

@@ -0,0 +1,5 @@
+const { crawlAll } = require("./crawl");
+
+(async () => {
+  await crawlAll();
+})();

app/public/main.css

@@ -7,7 +7,6 @@
   left: 25%;
 }
 
-
 .welcome-big-logo {
   font-size: 200pt;
   background-image: url(./images/logo.png);
@@ -16,19 +15,17 @@
   color: rgba(0, 0, 0, 0);
 }
 .no-ui-slider {
-  width: 95%
+  width: 95%;
 }
 
 .centered-element {
   margin-top: 200px;
 }
 
-
 .centered-element-small {
   margin-top: 100px;
 }
 
-
 #map {
   height: 50%;
 }
@@ -38,7 +35,7 @@ body {
   height: 100%;
   margin: 0;
   padding: 0;
-  font-family: 'Tangerine', serif;
+  font-family: "Tangerine", serif;
 }
 
 #floating-panel {
@@ -49,7 +46,7 @@ body {
   padding: 5px;
   border: 1px solid #999;
   text-align: center;
-  font-family: 'Roboto', 'sans-serif';
+  font-family: "Roboto", "sans-serif";
   line-height: 30px;
   padding-left: 10px;
 }

development.env

@@ -14,6 +14,9 @@ AMAZON_REGION=eu-west-1
 APP_URL=http://localhost:3001
 SOURCE_EMAIL=info@saburly.com
 
+CRAWLER_INTERVAL=Interval to run cralwer(s), in seconds
+STOP_CRAWLER=Non-zero value will skip crawler execution
+
 #=============== CRAWLER SETTINGS===============#
 #==OLX==
 OLX_MAX_PAGES=Restrict crawler to this number of pages

index.js

@@ -6,8 +6,13 @@ const bodyParser = require("body-parser");
 const layout = require("express-layout");
 const compression = require("compression");
 
-const { APP_PORT } = require("./app/config/appConfig");
+const {
+  APP_PORT,
+  CRAWLER_INTERVAL,
+  STOP_CRAWLER
+} = require("./app/config/appConfig");
 const routes = require("./app/routes");
+const { crawlAll } = require("./app/crawler/crawl");
 
 const app = express();
 
@@ -26,3 +31,14 @@ app.use("/assets", express.static("./app/public"));
 app.listen(APP_PORT, () =>
   console.log(`Example app listening on port ${APP_PORT}!`)
 );
+
+let crawlerRunning = STOP_CRAWLER;
+const crawl = () => {
+  if (!crawlerRunning) {
+    crawlerRunning = true;
+    crawlAll().then(newRealEstates => {
+      crawlerRunning = false;
+    });
+  }
+};
+setInterval(crawl, CRAWLER_INTERVAL * 1000);

package.json

@@ -11,7 +11,7 @@
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",
-    "crawl": "cd app/crawler && node crawl.js"
+    "crawl": "cd app/crawler && node npmCrawl.js"
   },
   "repository": {
     "type": "git",