crawling and saving to mongo

crawler/crawl.js

@@ -24,25 +24,29 @@ let savers = [
   new MongoSaver(process.env.MONGO_URL)
 ];
 
-let done = 0;
-for (let crawler of crawlers) {
-  try {
-    crawler.crawl().then((results) => {
+async function crawlAll() {
+  let crawlersAndSaversDone = 0;
+
+  for (let crawler of crawlers) {
+    try {
+      let results = await crawler.crawl()
       for (let saver of savers) {
         try {
-          saver.open();
-          saver.save(results);
+          await saver.connect();
+          await saver.save(results);
         } catch (e) {
           console.log("Error saving. Trying next saver! ", e);
-
+          crawlersAndSaversDone += 1;
         }
       }
-    });
-  } catch (e) {
-    console.log("Error crawling. Trying next crawler! ", e);
+    } catch (e) {
+      console.log("Error crawling. Trying next crawler! ", e);
+    }
+  }
+
+  for (let saver of savers) {
+    saver.close();
   }
 }
 
-for (let saver of savers) {
-  saver.close();
-}
+crawlAll();