create new crawler and Postgres saver

app/crawler/crawl.js

@@ -8,48 +8,28 @@
 
 require("dotenv").config();
 const OlxCrawler = require("./specific/olx");
+const { OLX_CONFIG } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
-let crawlers = [
-  // new OlxCrawler(
-  //   process.env.OLX_FROM_PAGE,
-  //   process.env.OLX_TO_PAGE,
-  //   process.env.OLX_MAX_RESULTS
-  // )
-  // new ProstorCrawler(
-  //   parseInt(process.env.PROSTOR_FROM_PAGE),
-  //   parseInt(process.env.PROSTOR_TO_PAGE),
-  //   parseInt(process.env.PROSTOR_MAX_RESULTS)
-  // ),
-  // new RentalCrawler(
-  //   parseInt(process.env.RENTAL_FROM_PAGE),
-  //   parseInt(process.env.RENTAL_TO_PAGE),
-  //   parseInt(process.env.RENTAL_MAX_RESULTS)
-  // )
+const crawlers = [
+  new OlxCrawler(
+    OLX_CONFIG.OLX_START_PAGE,
+    OLX_CONFIG.OLX_END_PAGE,
+    OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
+    [new PostgresSaver()],
+    OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
+    OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES
+  )
 ];
 
-let savers = [new PostgresSaver(process.env.MONGO_URL)];
-
 async function crawlAll() {
   for (let crawler of crawlers) {
     try {
-      const crawlerResults = await crawler.crawl();
-      for (let saver of savers) {
-        try {
-          await saver.connect();
-          await saver.save(crawlerResults);
-        } catch (e) {
-          console.log("Error saving. Trying next saver! ", e);
-        }
-      }
+      await crawler.crawl();
     } catch (e) {
       console.log("Error crawling. Trying next crawler! ", e);
     }
   }
-
-  for (let saver of savers) {
-    saver.close();
-  }
 }
 
 crawlAll();