move specific crawler config to the separated files

app/crawler/crawl.js

@@ -5,8 +5,8 @@
    All environment specific configuration is read here and
    passed to the crawlers and savers.
 */
-const OlxCrawler = require("./specific/olx");
+const OlxCrawler = require("./specificCrawlers/olx");
-const RentalCrawler = require("./specific/rental");
+const RentalCrawler = require("./specificCrawlers/rental");
 
 const { OLX_CONFIG, RENTAL_CONFIG } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");

app/crawler/crawlerConfig.js

@@ -1,72 +1,8 @@
 "use strict";
 require("dotenv").config({ path: __dirname + "/./../../.env" });
-const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../common/enums");
 
-const olxCrawlerAdType =
+const OLX_CONFIG = require("./specificConfigs/olx");
-  process.env.OLX_CRAWLER_AD_TYPE !== undefined
+const RENTAL_CONFIG = require("./specificConfigs/rental");
-    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
-    : null;
-
-const rentalCrawlerAdType =
-  process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
-    ? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
-    : null;
-
-const olxParsedCrawlerAdCategories =
-  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
-    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
-        category.trim()
-      )
-    : ["FLAT", "HOUSE"];
-
-const rentalParsedCrawlerAdCategories =
-  process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
-    ? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
-        category.trim()
-      )
-    : ["FLAT", "HOUSE"];
-
-const olxIgnoredUsernames =
-  process.env.OLX_IGNORED_USERNAMES !== undefined
-    ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
-        username.trim()
-      )
-    : [];
-
-const rentalIgnoredUsernames = [];
-
-const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
-  .map(categoryName =>
-    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
-  )
-  .filter(category => !!category);
-
-const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
-  .map(categoryName =>
-    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
-  )
-  .filter(category => !!category);
-
-const OLX_CONFIG = {
-  OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
-  OLX_MAX_RESULTS_PER_PAGE:
-    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
-  OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
-  OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
-  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
-  OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
-};
-
-const RENTAL_CONFIG = {
-  RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
-  RENTAL_MAX_RESULTS_PER_PAGE:
-    parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
-  RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
-  RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
-  RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
-  RENTAL_DELAY_BETWEEN_PAGES:
-    parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000
-};
 
 module.exports = {
   OLX_CONFIG,

app/crawler/specificConfigs/olx.js

@@ -0,0 +1,37 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const olxCrawlerAdType =
+  process.env.OLX_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
+    : null;
+
+const olxParsedCrawlerAdCategories =
+  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const olxIgnoredUsernames =
+  process.env.OLX_IGNORED_USERNAMES !== undefined
+    ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
+        username.trim()
+      )
+    : [];
+
+const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
+  OLX_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
+  OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
+  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
+  OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
+};

app/crawler/specificConfigs/rental.js

@@ -0,0 +1,33 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const rentalCrawlerAdType =
+  process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
+    : null;
+
+const rentalParsedCrawlerAdCategories =
+  process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const rentalIgnoredUsernames = [];
+
+const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
+  RENTAL_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
+  RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
+  RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
+  RENTAL_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000
+};

test/olxScrapeTest.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const olxCrawler = require("../app/crawler/specific/olx");
+const olxCrawler = require("../app/crawler/specificCrawlers/olx");
 
 const urlToScrape = process.argv[2] || undefined;