senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Cleanup after debugging

Browse Source
This commit is contained in:
=
2020-09-16 06:16:49 -07:00
parent bf8d131025
commit 2b1cbcaa47
4 changed files with 43 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
app/crawler/crawl.js
View File

@@ -33,44 +33,43 @@ async function crawlAll() {
OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
OLX_CONFIG.OLX_IGNORED_USERNAMES,
OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
),
new RentalCrawler(
[postgresSaver],
RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
RENTAL_CONFIG.RENTAL_MAX_PAGES,
RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
),
new ProstorCrawler(
[postgresSaver],
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
),
new AktidoCrawler(
[postgresSaver],
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
),
new SaljicCrawler(
[postgresSaver],
SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
SALJIC_CONFIG.SALJIC_MAX_PAGES,
SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
)
//new RentalCrawler(
//[postgresSaver],
//RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
//RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
//RENTAL_CONFIG.RENTAL_MAX_PAGES,
//RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
//RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
//RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
//),
//new ProstorCrawler(
//[postgresSaver],
//PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
//PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
//PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
//PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
//PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
//PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
//),
//new AktidoCrawler(
//[postgresSaver],
//AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
//AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
//AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
//AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
//AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
//AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
//)
//,
//new SaljicCrawler(
//[postgresSaver],
//SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
//SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
//SALJIC_CONFIG.SALJIC_MAX_PAGES,
//SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
//SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
//SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
//)
];
const newRealEstates = [];