senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

implement crawler for Prostor agency

Browse Source
This commit is contained in:
Bilal Catic
2019-10-25 10:54:08 +02:00
parent 05fad652c4
commit 7e3b0bfcd5
3 changed files with 264 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/crawler/specificConfigs/prostor.js
View File

@@ -24,7 +24,7 @@ const transformedProstorCrawlerAdCategories = prostorParsedCrawlerAdCategories
module.exports = {
PROSTOR_MAX_PAGES: parseInt(process.env.PROSTOR_MAX_PAGES) || 100,
PROSTOR_MAX_RESULTS_PER_PAGE:
parseInt(process.env.PROSTOR_MAX_RESULTS_PER_PAGE) || 50,
parseInt(process.env.PROSTOR_MAX_RESULTS_PER_PAGE) || 5000,
PROSTOR_CRAWLER_AD_TYPE: prostorCrawlerAdType || CRAWLER_AD_TYPE.NONE,
PROSTOR_CRAWLER_AD_CATEGORIES: transformedProstorCrawlerAdCategories,
PROSTOR_IGNORED_USERNAMES: prostorIgnoredUsernames || [],