create new crawler and Postgres saver
This commit is contained in:
32
app/crawler/crawlerConfig.js
Normal file
32
app/crawler/crawlerConfig.js
Normal file
@@ -0,0 +1,32 @@
|
||||
"use strict";
|
||||
require("dotenv").config({ path: "../../.env" });
|
||||
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../common/enums");
|
||||
|
||||
const crawlerAdType =
|
||||
process.env.OLX_CRAWLER_AD_TYPE !== undefined
|
||||
? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
|
||||
: null;
|
||||
|
||||
const parsedCrawlerAdCategories =
|
||||
process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
|
||||
? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
|
||||
category.trim()
|
||||
)
|
||||
: ["CATEGORY_FLAT", "CATEGORY_HOUSE"];
|
||||
|
||||
const transformedCrawlerAdCategories = parsedCrawlerAdCategories
|
||||
.map(categoryName => AD_CATEGORY[categoryName])
|
||||
.filter(category => !!category);
|
||||
|
||||
const OLX_CONFIG = {
|
||||
OLX_START_PAGE: parseInt(process.env.OLX_START_PAGE) || 1,
|
||||
OLX_END_PAGE: parseInt(process.env.OLX_END_PAGE) || 10,
|
||||
OLX_MAX_RESULTS_PER_PAGE:
|
||||
parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
|
||||
OLX_CRAWLER_AD_TYPE: crawlerAdType || CRAWLER_AD_TYPE.NONE,
|
||||
OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
OLX_CONFIG
|
||||
};
|
||||
Reference in New Issue
Block a user