senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

move ignored usernames config to crawler specific config

Browse Source
This commit is contained in:
Bilal Catic
2019-09-24 23:21:06 +02:00
parent 63eb64b0f6
commit 06d35fcb4b
2 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/common/enums.js
View File

@@ -12,8 +12,6 @@ const AD_CATEGORY = {
CATEGORY_GARAGE: "GARAGE" CATEGORY_GARAGE: "GARAGE"
}; };
const IGNORED_USERNAMES = [];
const AD_STATUS = { const AD_STATUS = {
STATUS_NORMAL: 1, STATUS_NORMAL: 1,
STATUS_RESERVED: 2, STATUS_RESERVED: 2,

11
app/crawler/crawlerConfig.js
View File

@@ -14,17 +14,24 @@ const olxParsedCrawlerAdCategories =
) )
: ["CATEGORY_FLAT", "CATEGORY_HOUSE"]; : ["CATEGORY_FLAT", "CATEGORY_HOUSE"];
const olxIgnoredUsernames =
process.env.OLX_IGNORED_USERNAMES !== undefined
? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
username.trim()
)
: [];
const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories
.map(categoryName => AD_CATEGORY[categoryName]) .map(categoryName => AD_CATEGORY[categoryName])
.filter(category => !!category); .filter(category => !!category);
const OLX_CONFIG = { const OLX_CONFIG = {
OLX_MAX_PAGES: parseInt(process.env.MAX_PAGES) || 500, OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
OLX_MAX_RESULTS_PER_PAGE: OLX_MAX_RESULTS_PER_PAGE:
parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50, parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE, OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories, OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories,
OLX_MAX_AGE: parseInt(process.env.OLX_MAX_AGE) || 30 OLX_IGNORED_USERNAMES: olxIgnoredUsernames || []
}; };
module.exports = { module.exports = {