diff --git a/app/config/appConfig.js b/app/config/appConfig.js index 5b06652..288ee24 100644 --- a/app/config/appConfig.js +++ b/app/config/appConfig.js @@ -8,8 +8,13 @@ const APP_URL = const DEFAULT_TIMEZONE = "Europe/Sarajevo"; +const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60; +const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER); + module.exports = { APP_PORT, APP_URL, - DEFAULT_TIMEZONE + DEFAULT_TIMEZONE, + CRAWLER_INTERVAL, + STOP_CRAWLER }; diff --git a/app/crawler/crawl.js b/app/crawler/crawl.js index 77d4fc9..e77053c 100644 --- a/app/crawler/crawl.js +++ b/app/crawler/crawl.js @@ -26,15 +26,14 @@ const crawlers = [ async function crawlAll() { for (let crawler of crawlers) { try { - const newRealEstates = await crawler.crawl(); - - console.log("Number of new real estates : ", newRealEstates.length); + return await crawler.crawl(); } catch (e) { console.log("Error crawling. Trying next crawler! ", e); + return []; } } } -(async () => { - await crawlAll(); -})(); +module.exports = { + crawlAll +}; diff --git a/app/crawler/npmCrawl.js b/app/crawler/npmCrawl.js new file mode 100644 index 0000000..fa26ebc --- /dev/null +++ b/app/crawler/npmCrawl.js @@ -0,0 +1,5 @@ +const { crawlAll } = require("./crawl"); + +(async () => { + await crawlAll(); +})(); diff --git a/app/public/main.css b/app/public/main.css index b936827..84f6863 100644 --- a/app/public/main.css +++ b/app/public/main.css @@ -1,14 +1,13 @@ -.welcome-center-button { - width: 100%; +.welcome-center-button { + width: 100%; } -.next-center-button { +.next-center-button { width: 50%; - left: 25%; + left: 25%; } - -.welcome-big-logo { +.welcome-big-logo { font-size: 200pt; background-image: url(./images/logo.png); background-size: contain; @@ -16,19 +15,17 @@ color: rgba(0, 0, 0, 0); } .no-ui-slider { - width: 95% + width: 95%; } .centered-element { margin-top: 200px; } - .centered-element-small { margin-top: 100px; } - #map { height: 50%; } @@ -38,9 +35,9 @@ body { height: 100%; margin: 0; padding: 0; - font-family: 'Tangerine', serif; + font-family: "Tangerine", serif; } - + #floating-panel { top: 10px; left: 25%; @@ -49,7 +46,7 @@ body { padding: 5px; border: 1px solid #999; text-align: center; - font-family: 'Roboto', 'sans-serif'; + font-family: "Roboto", "sans-serif"; line-height: 30px; padding-left: 10px; } diff --git a/development.env b/development.env index 2f2fdc0..480f5a5 100644 --- a/development.env +++ b/development.env @@ -14,6 +14,9 @@ AMAZON_REGION=eu-west-1 APP_URL=http://localhost:3001 SOURCE_EMAIL=info@saburly.com +CRAWLER_INTERVAL=Interval to run cralwer(s), in seconds +STOP_CRAWLER=Non-zero value will skip crawler execution + #=============== CRAWLER SETTINGS===============# #==OLX== OLX_MAX_PAGES=Restrict crawler to this number of pages diff --git a/index.js b/index.js index c1a7b7d..5faf461 100644 --- a/index.js +++ b/index.js @@ -6,8 +6,13 @@ const bodyParser = require("body-parser"); const layout = require("express-layout"); const compression = require("compression"); -const { APP_PORT } = require("./app/config/appConfig"); +const { + APP_PORT, + CRAWLER_INTERVAL, + STOP_CRAWLER +} = require("./app/config/appConfig"); const routes = require("./app/routes"); +const { crawlAll } = require("./app/crawler/crawl"); const app = express(); @@ -26,3 +31,14 @@ app.use("/assets", express.static("./app/public")); app.listen(APP_PORT, () => console.log(`Example app listening on port ${APP_PORT}!`) ); + +let crawlerRunning = STOP_CRAWLER; +const crawl = () => { + if (!crawlerRunning) { + crawlerRunning = true; + crawlAll().then(newRealEstates => { + crawlerRunning = false; + }); + } +}; +setInterval(crawl, CRAWLER_INTERVAL * 1000); diff --git a/package.json b/package.json index f55a068..f915d34 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate", "docker-start": "docker start pg_marketalerts", "docker-stop": "docker stop pg_marketalerts", - "crawl": "cd app/crawler && node crawl.js" + "crawl": "cd app/crawler && node npmCrawl.js" }, "repository": { "type": "git",