senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Add crawler for rental page #58

Merged
bilal.catic merged 3 commits from add-crawler-for-rental-page into master 2019-10-24 15:20:41 +02:00
Conversation 3 Commits 3 Files Changed 5 +25 -18
1 changed files with 25 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 2064d40985 - Show all commits

43
app/crawler/specific/rental.js
View File

@@ -80,27 +80,31 @@ class RentalCrawler {
for (const [index, { value: singlePageResult }] of entries) {
if (singlePageResult) {
const saveResults = await this.saveCrawledResults(singlePageResult);
const { newRecords, existingRecords } = saveResults;
const { newRecords } = saveResults;
newRealEstates.push(...newRecords);
for (const existingRecord of existingRecords) {
const { publishedDate, renewedDate } = existingRecord;
const publishedDateMoment = moment.utc(publishedDate);
const renewedDateMoment = moment.utc(renewedDate);
const stopCrawlingThisCategory = publishedDateMoment.isSame(
renewedDateMoment,
"minute"
);
if (stopCrawlingThisCategory) {
generatorsToRemove[index] = true;
// console.log("\tGenerator ", index + 1, "has no more new ads");
break;
}
if (Array.isArray(newRecords) && newRecords.length === 0) {
generatorsToRemove[index] = true;
}
// for (const existingRecord of existingRecords) {
// const { publishedDate, renewedDate } = existingRecord;
//
// const publishedDateMoment = moment.utc(publishedDate);
// const renewedDateMoment = moment.utc(renewedDate);
//
// const stopCrawlingThisCategory = publishedDateMoment.isSame(
// renewedDateMoment,
// "minute"
// );
//
// if (stopCrawlingThisCategory) {
// generatorsToRemove[index] = true;
// // console.log("\tGenerator ", index + 1, "has no more new ads");
// break;
// }
// }
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;
@@ -240,7 +244,10 @@ class RentalCrawler {
}
const title = extractedData["re_realEstates_portalName"];
const price = parseFloat(extractedData["re_realEstates_price"]);
const extractedPrice = parseFloat(
extractedData["re_realEstates_price"]
);
const price = extractedPrice ? extractedPrice : null;
const area = parseFloat(extractedData["re_realEstates_area"]);
const gardenSize = parseFloat(
extractedData["re_realEstates_fieldArea"]