old-web/app/crawler/crawlerConfig.js

"use strict";
require("dotenv").config({ path: __dirname + "/./../../.env" });
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../common/enums");

const olxCrawlerAdType =
  process.env.OLX_CRAWLER_AD_TYPE !== undefined
    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
    : null;

const olxParsedCrawlerAdCategories =
  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
        category.trim()
      )
    : ["FLAT", "HOUSE"];

const olxIgnoredUsernames =
  process.env.OLX_IGNORED_USERNAMES !== undefined
    ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
        username.trim()
      )
    : [];

const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories
  .map(categoryName =>
    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
  )
  .filter(category => !!category);

const OLX_CONFIG = {
  OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
  OLX_MAX_RESULTS_PER_PAGE:
    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
  OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
  OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories,
  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
  OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
};

module.exports = {
  OLX_CONFIG
};