crawlers and savers

crawler/crawl.js

@@ -0,0 +1,48 @@
+'use strict'
+
+
+/*
+   Entry point for crawling functionality
+   All communication between crawlers and savers is here
+   All environment specific configuration is read here and
+   passed to the crawlers and savers.
+
+*/
+import {
+  install
+} from 'source-map-support';
+import 'dotenv/config';
+import OlxCrawler from './specific/olx';
+import MongoSaver from './savers/mongo'
+
+install(); // for source maps to work
+
+let crawlers = [
+  new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS)
+];
+let savers = [
+  new MongoSaver(process.env.MONGO_URL)
+];
+
+let done = 0;
+for (let crawler of crawlers) {
+  try {
+    crawler.crawl().then((results) => {
+      for (let saver of savers) {
+        try {
+          saver.open();
+          saver.save(results);
+        } catch (e) {
+          console.log("Error saving. Trying next saver! ", e);
+
+        }
+      }
+    });
+  } catch (e) {
+    console.log("Error crawling. Trying next crawler! ", e);
+  }
+}
+
+for (let saver of savers) {
+  saver.close();
+}