senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

More debugging, disable all crawlers

Browse Source
This commit is contained in:
Edin Dazdarevic
2020-09-13 23:45:08 +02:00
parent dfbefc20cd
commit d871d9ad1f
2 changed files with 31 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
app/crawler/crawl.js
View File

@@ -10,6 +10,7 @@ const RentalCrawler = require("./specificCrawlers/rental");
const ProstorCrawler = require("./specificCrawlers/prostor"); const ProstorCrawler = require("./specificCrawlers/prostor");
const AktidoCrawler = require("./specificCrawlers/aktido"); const AktidoCrawler = require("./specificCrawlers/aktido");
const SaljicCrawler = require("./specificCrawlers/saljic"); const SaljicCrawler = require("./specificCrawlers/saljic");
const { logDebug } = require("../helpers/log");
const { const {
OLX_CONFIG, OLX_CONFIG,
@@ -32,34 +33,34 @@ async function crawlAll() {
OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE, OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
OLX_CONFIG.OLX_IGNORED_USERNAMES, OLX_CONFIG.OLX_IGNORED_USERNAMES,
OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
),
new RentalCrawler(
[postgresSaver],
RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
RENTAL_CONFIG.RENTAL_MAX_PAGES,
RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
),
new ProstorCrawler(
[postgresSaver],
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
),
new AktidoCrawler(
[postgresSaver],
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
) )
//new RentalCrawler(
//[postgresSaver],
//RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
//RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
//RENTAL_CONFIG.RENTAL_MAX_PAGES,
//RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
//RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
//RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
//),
//new ProstorCrawler(
//[postgresSaver],
//PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
//PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
//PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
//PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
//PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
//PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
//),
//new AktidoCrawler(
//[postgresSaver],
//AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
//AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
//AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
//AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
//AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
//AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
//)
//, //,
//new SaljicCrawler( //new SaljicCrawler(
//[postgresSaver], //[postgresSaver],
@@ -76,7 +77,9 @@ async function crawlAll() {
for (const crawler of crawlers) { for (const crawler of crawlers) {
try { try {
logDebug('Starting crawler: ', crawler);
const newRealEstatesFromSingleCrawler = await crawler.crawl(); const newRealEstatesFromSingleCrawler = await crawler.crawl();
logDebug('Crawler done: ', crawler);
if (Array.isArray(newRealEstatesFromSingleCrawler)) { if (Array.isArray(newRealEstatesFromSingleCrawler)) {
newRealEstates.push(...newRealEstatesFromSingleCrawler); newRealEstates.push(...newRealEstatesFromSingleCrawler);
} }

2
index.js
View File

@@ -39,7 +39,7 @@ app.listen(APP_PORT, () =>
let crawlerRunning = STOP_CRAWLER; let crawlerRunning = STOP_CRAWLER;
const crawl = () => { const crawl = () => {
logDebug("Crawl start"); logDebug("Crawl start. crawlerRunning: ", crawlerRunning);
if (!crawlerRunning) { if (!crawlerRunning) {
crawlerRunning = true; crawlerRunning = true;
crawlAll().then(newRealEstates => { crawlAll().then(newRealEstates => {