'use strict'


/*
   Entry point for crawling functionality
   All communication between crawlers and savers is here
   All environment specific configuration is read here and
   passed to the crawlers and savers.

*/
import {
  install
} from 'source-map-support';
import 'dotenv/config';
import OlxCrawler from './specific/olx';
import MongoSaver from './savers/mongo'

install(); // for source maps to work

let crawlers = [
  new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS)
];
let savers = [
  new MongoSaver(process.env.MONGO_URL)
];

let done = 0;
for (let crawler of crawlers) {
  try {
    crawler.crawl().then((results) => {
      for (let saver of savers) {
        try {
          saver.open();
          saver.save(results);
        } catch (e) {
          console.log("Error saving. Trying next saver! ", e);

        }
      }
    });
  } catch (e) {
    console.log("Error crawling. Trying next crawler! ", e);
  }
}

for (let saver of savers) {
  saver.close();
}