use function generator to index pages; crawl in parallel

2019-09-23 10:46:31 +02:00
parent c4f6c6e1c3
commit 3140fdf0c0
6 changed files with 127 additions and 77 deletions
--- a/app/crawler/crawl.js
+++ b/app/crawler/crawl.js
@@ -13,12 +13,12 @@ const PostgresSaver = require("./savers/postgres");

 const crawlers = [
  new OlxCrawler(
-    OLX_CONFIG.OLX_START_PAGE,
-    OLX_CONFIG.OLX_END_PAGE,
-    OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
    [new PostgresSaver()],
    OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
-    OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES
+    OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
+    OLX_CONFIG.OLX_MAX_PAGES,
+    OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
+    OLX_CONFIG.OLX_MAX_AGE
  )
 ];