senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Implement renting option - crawler part #64

Merged
bilal.catic merged 5 commits from implement-renting-option into master 2019-10-31 19:05:55 +01:00
Conversation 5 Commits 5 Files Changed 7 +9 -20
2 changed files with 9 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fa712ce97d - Show all commits

3
app/crawler/specificConfigs/rental.js
View File

@@ -29,5 +29,6 @@ module.exports = {
RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
RENTAL_DELAY_BETWEEN_PAGES:
parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000
parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000,
RENTAL_FORCE_CRAWL: !!parseInt(process.env.RENTAL_FORCE_CRAWL)
};

26
app/crawler/specificCrawlers/rental.js
View File

@@ -39,6 +39,8 @@ const RENTAL_ENUMS = {
RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
};
const { RENTAL_FORCE_CRAWL } = require("../specificConfigs/rental");
class RentalCrawler {
constructor(
savers = [],
@@ -88,27 +90,13 @@ class RentalCrawler {
newRealEstates.push(...newRecords);
if (Array.isArray(newRecords) && newRecords.length === 0) {
if (
Array.isArray(newRecords) &&
newRecords.length === 0 &&
!RENTAL_FORCE_CRAWL
) {
generatorsToRemove[index] = true;
}
// for (const existingRecord of existingRecords) {
// const { publishedDate, renewedDate } = existingRecord;
//
// const publishedDateMoment = moment.utc(publishedDate);
// const renewedDateMoment = moment.utc(renewedDate);
//
// const stopCrawlingThisCategory = publishedDateMoment.isSame(
// renewedDateMoment,
// "minute"
// );
//
// if (stopCrawlingThisCategory) {
// generatorsToRemove[index] = true;
// // console.log("\tGenerator ", index + 1, "has no more new ads");
// break;
// }
// }
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;