diff --git a/app/crawler/specificConfigs/rental.js b/app/crawler/specificConfigs/rental.js index 8930d64..103723e 100644 --- a/app/crawler/specificConfigs/rental.js +++ b/app/crawler/specificConfigs/rental.js @@ -29,5 +29,6 @@ module.exports = { RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories, RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [], RENTAL_DELAY_BETWEEN_PAGES: - parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000 + parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000, + RENTAL_FORCE_CRAWL: !!parseInt(process.env.RENTAL_FORCE_CRAWL) }; diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js index b73278e..6293d1f 100644 --- a/app/crawler/specificCrawlers/rental.js +++ b/app/crawler/specificCrawlers/rental.js @@ -39,6 +39,8 @@ const RENTAL_ENUMS = { RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss" }; +const { RENTAL_FORCE_CRAWL } = require("../specificConfigs/rental"); + class RentalCrawler { constructor( savers = [], @@ -88,27 +90,13 @@ class RentalCrawler { newRealEstates.push(...newRecords); - if (Array.isArray(newRecords) && newRecords.length === 0) { + if ( + Array.isArray(newRecords) && + newRecords.length === 0 && + !RENTAL_FORCE_CRAWL + ) { generatorsToRemove[index] = true; } - - // for (const existingRecord of existingRecords) { - // const { publishedDate, renewedDate } = existingRecord; - // - // const publishedDateMoment = moment.utc(publishedDate); - // const renewedDateMoment = moment.utc(renewedDate); - // - // const stopCrawlingThisCategory = publishedDateMoment.isSame( - // renewedDateMoment, - // "minute" - // ); - // - // if (stopCrawlingThisCategory) { - // generatorsToRemove[index] = true; - // // console.log("\tGenerator ", index + 1, "has no more new ads"); - // break; - // } - // } } else { //Generator returned undefined, remove this generator from array generatorsToRemove[index] = true;