old-kivi/crawler/crawl.js

'use strict'


/*
   Entry point for crawling functionality
   All communication between crawlers and savers is here
   All environment specific configuration is read here and
   passed to the crawlers and savers.

*/
import {
  install
} from 'source-map-support';
import 'dotenv/config';
import OlxCrawler from './specific/olx';
import ProstorCrawler from './specific/prostor';
import RentalCrawler from './specific/rental';
import MongoSaver from './savers/mongo'

install(); // for source maps to work

let crawlers = [
  //new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
  new ProstorCrawler(parseInt(process.env.PROSTOR_FROM_PAGE), parseInt(process.env.PROSTOR_TO_PAGE), parseInt(process.env.PROSTOR_MAX_RESULTS)),
  new RentalCrawler(parseInt(process.env.RENTAL_FROM_PAGE), parseInt(process.env.RENTAL_TO_PAGE), parseInt(process.env.RENTAL_MAX_RESULTS))
];

let savers = [
  new MongoSaver(process.env.MONGO_URL)
];

async function crawlAll() {

  for (let crawler of crawlers) {
    try {
      let results = await crawler.crawl()
      for (let saver of savers) {
        try {
          await saver.connect();
          await saver.save(results);
        } catch (e) {
          console.log("Error saving. Trying next saver! ", e);
        }
      }
    } catch (e) {
      console.log("Error crawling. Trying next crawler! ", e);
    }
  }

  for (let saver of savers) {
    saver.close();
  }
}

crawlAll();