"use strict";
/*
   Entry point for crawling functionality
   All communication between crawlers and savers is here
   All environment specific configuration is read here and
   passed to the crawlers and savers.
*/
const OlxCrawler = require("./specific/olx");
const RentalCrawler = require("./specific/rental");

const { OLX_CONFIG, RENTAL_CONFIG } = require("./crawlerConfig");
const PostgresSaver = require("./savers/postgres");

async function crawlAll() {
  const postgresSaver = new PostgresSaver();

  const crawlers = [
    new OlxCrawler(
      [postgresSaver],
      OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
      OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
      OLX_CONFIG.OLX_MAX_PAGES,
      OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
      OLX_CONFIG.OLX_IGNORED_USERNAMES,
      OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
    ),
    new RentalCrawler(
      [postgresSaver],
      RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
      RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
      RENTAL_CONFIG.RENTAL_MAX_PAGES,
      RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
      RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
      RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
    )
  ];

  const newRealEstates = [];

  for (const crawler of crawlers) {
    try {
      const newRealEstatesFromSingleCrawler = await crawler.crawl();
      if (Array.isArray(newRealEstatesFromSingleCrawler)) {
        newRealEstates.push(...newRealEstatesFromSingleCrawler);
      }
    } catch (e) {
      console.log("Error crawling. Trying next crawler! ", e);
    }
  }

  return newRealEstates;
}

module.exports = {
  crawlAll
};