old-web/app/crawler/crawl.js

"use strict";
/*
   Entry point for crawling functionality
   All communication between crawlers and savers is here
   All environment specific configuration is read here and
   passed to the crawlers and savers.
*/
const OlxCrawler = require("./specific/olx");
const { OLX_CONFIG } = require("./crawlerConfig");
const PostgresSaver = require("./savers/postgres");

const crawlers = [
  new OlxCrawler(
    [new PostgresSaver()],
    OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
    OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
    OLX_CONFIG.OLX_MAX_PAGES,
    OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
    OLX_CONFIG.OLX_IGNORED_USERNAMES,
    OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
  )
];

async function crawlAll() {
  for (let crawler of crawlers) {
    try {
      return await crawler.crawl();
    } catch (e) {
      console.log("Error crawling. Trying next crawler! ", e);
      return [];
    }
  }
}

module.exports = {
  crawlAll
};