start crawler loop when server is started

app/config/appConfig.js

@@ -8,8 +8,11 @@ const APP_URL =
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
+const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
+
 module.exports = {
   APP_PORT,
   APP_URL,
-  DEFAULT_TIMEZONE
+  DEFAULT_TIMEZONE,
+  CRAWLER_INTERVAL
 };

app/crawler/crawl.js

@@ -26,15 +26,14 @@ const crawlers = [
 async function crawlAll() {
   for (let crawler of crawlers) {
     try {
-      const newRealEstates = await crawler.crawl();
+      return await crawler.crawl();
-
-      console.log("Number of new real estates : ", newRealEstates.length);
     } catch (e) {
       console.log("Error crawling. Trying next crawler! ", e);
+      return [];
     }
   }
 }
 
-(async () => {
+module.exports = {
-  await crawlAll();
+  crawlAll
-})();
+};

app/crawler/npmCrawl.js

@@ -0,0 +1,5 @@
+const { crawlAll } = require("./crawl");
+
+(async () => {
+  await crawlAll();
+})();

development.env

@@ -14,6 +14,8 @@ AMAZON_REGION=eu-west-1
 APP_URL=http://localhost:3001
 SOURCE_EMAIL=info@saburly.com
 
+CRAWLER_INTERVAL=Interval to run cralwer(s), in seconds
+
 #=============== CRAWLER SETTINGS===============#
 #==OLX==
 OLX_MAX_PAGES=Restrict crawler to this number of pages

index.js

@@ -6,8 +6,9 @@ const bodyParser = require("body-parser");
 const layout = require("express-layout");
 const compression = require("compression");
 
-const { APP_PORT } = require("./app/config/appConfig");
+const { APP_PORT, CRAWLER_INTERVAL } = require("./app/config/appConfig");
 const routes = require("./app/routes");
+const { crawlAll } = require("./app/crawler/crawl");
 
 const app = express();
 
@@ -26,3 +27,14 @@ app.use("/assets", express.static("./app/public"));
 app.listen(APP_PORT, () =>
   console.log(`Example app listening on port ${APP_PORT}!`)
 );
+
+let crawlerRunning = false;
+const crawl = () => {
+  if (!crawlerRunning) {
+    crawlerRunning = true;
+    crawlAll().then(newRealEstates => {
+      crawlerRunning = false;
+    });
+  }
+};
+setInterval(crawl, CRAWLER_INTERVAL * 1000);

package.json

@@ -11,7 +11,7 @@
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",
-    "crawl": "cd app/crawler && node crawl.js"
+    "crawl": "cd app/crawler && node npmCrawl.js"
   },
   "repository": {
     "type": "git",