senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

add crawler config and include specific crawler for "rental" agency

Browse Source
This commit is contained in:
Bilal Catic
2019-10-24 07:48:49 +02:00
parent abc591749e
commit ec798fe94c
4 changed files with 80 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
app/crawler/crawlerConfig.js
View File

@@ -7,6 +7,11 @@ const olxCrawlerAdType =
? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
: null;
const rentalCrawlerAdType =
process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
: null;
const olxParsedCrawlerAdCategories =
process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
@@ -14,6 +19,13 @@ const olxParsedCrawlerAdCategories =
)
: ["FLAT", "HOUSE"];
const rentalParsedCrawlerAdCategories =
process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const olxIgnoredUsernames =
process.env.OLX_IGNORED_USERNAMES !== undefined
? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
@@ -21,7 +33,15 @@ const olxIgnoredUsernames =
)
: [];
const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories
const rentalIgnoredUsernames = [];
const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
@@ -32,11 +52,23 @@ const OLX_CONFIG = {
OLX_MAX_RESULTS_PER_PAGE:
parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories,
OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
};
module.exports = {
OLX_CONFIG
const RENTAL_CONFIG = {
RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
RENTAL_MAX_RESULTS_PER_PAGE:
parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
RENTAL_DELAY_BETWEEN_PAGES:
parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000
};
module.exports = {
OLX_CONFIG,
RENTAL_CONFIG
};