implement crawler for "rental" agency
This commit is contained in:
Bilal Catic
369
app/crawler/specific/rental.js
Normal file
369
app/crawler/specific/rental.js
Normal file
@@ -0,0 +1,369 @@
|
||||
"use strict";
|
||||
|
||||
const fetch = require("node-fetch");
|
||||
const cheerio = require("cheerio");
|
||||
const Promise = require("bluebird");
|
||||
const moment = require("moment-timezone");
|
||||
const htmlToText = require("html-to-text");
|
||||
|
||||
const {
|
||||
AD_TYPE,
|
||||
AD_CATEGORY,
|
||||
AD_AGENCY,
|
||||
AD_STATUS,
|
||||
CRAWLER_AD_TYPE
|
||||
} = require("../../common/enums");
|
||||
|
||||
const { DEFAULT_TIMEZONE } = require("../../config/appConfig");
|
||||
|
||||
const RENTAL_ENUMS = {
|
||||
RENTAL_AD_TYPE: {
|
||||
[CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
|
||||
[CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
|
||||
[CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
|
||||
},
|
||||
RENTAL_AD_CATEGORY: {
|
||||
[AD_CATEGORY.FLAT.id]: "/tip-2",
|
||||
[AD_CATEGORY.HOUSE.id]: "/tip-1",
|
||||
[AD_CATEGORY.LAND.id]: "/tip-5",
|
||||
[AD_CATEGORY.OFFICE.id]: "/tip-4",
|
||||
[AD_CATEGORY.APARTMENT.id]: "/tip-3",
|
||||
[AD_CATEGORY.GARAGE.id]: "/tip-6"
|
||||
//[AD_CATEGORY.COTTAGE.id]: ""
|
||||
},
|
||||
RENTAL_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
|
||||
RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
|
||||
};
|
||||
|
||||
class RentalCrawler {
|
||||
constructor(
|
||||
savers = [],
|
||||
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
|
||||
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
|
||||
maxPages = 1000,
|
||||
maxResultsPerPage = 100,
|
||||
ignoredUsernames = [],
|
||||
delayBetweenPages = 1000
|
||||
) {
|
||||
this.savers = savers;
|
||||
this.baseUrl = "https://www.rental.ba/pretraga/sortiraj-date_DESC";
|
||||
this.crawlerAdTypes = crawlerAdTypes;
|
||||
this.crawlerAdCategories = crawlerAdCategories;
|
||||
this.maxPages = maxPages;
|
||||
this.maxResultsPerPage = maxResultsPerPage;
|
||||
this.delayBetweenPages = delayBetweenPages;
|
||||
}
|
||||
|
||||
async crawl() {
|
||||
const crawlAdCategories = this.crawlerAdCategories;
|
||||
|
||||
const newRealEstates = [];
|
||||
|
||||
if (crawlAdCategories) {
|
||||
const indexGenerators = [];
|
||||
for (const adCategory of crawlAdCategories) {
|
||||
indexGenerators.push(this.categoryIndexer(adCategory));
|
||||
}
|
||||
|
||||
let done = false;
|
||||
while (!done) {
|
||||
const categoryIndexerPromises = [];
|
||||
const generatorsToRemove = [];
|
||||
for (const indexGenerator of indexGenerators) {
|
||||
categoryIndexerPromises.push(indexGenerator.next());
|
||||
generatorsToRemove.push(false);
|
||||
}
|
||||
|
||||
const singlePageResults = await Promise.all(categoryIndexerPromises);
|
||||
const entries = singlePageResults.entries();
|
||||
|
||||
for (const [index, { value: singlePageResult }] of entries) {
|
||||
if (singlePageResult) {
|
||||
const saveResults = await this.saveCrawledResults(singlePageResult);
|
||||
const { newRecords, existingRecords } = saveResults;
|
||||
|
||||
newRealEstates.push(...newRecords);
|
||||
|
||||
for (const existingRecord of existingRecords) {
|
||||
const { publishedDate, renewedDate } = existingRecord;
|
||||
|
||||
const publishedDateMoment = moment.utc(publishedDate);
|
||||
const renewedDateMoment = moment.utc(renewedDate);
|
||||
|
||||
const stopCrawlingThisCategory = publishedDateMoment.isSame(
|
||||
renewedDateMoment,
|
||||
"minute"
|
||||
);
|
||||
|
||||
if (stopCrawlingThisCategory) {
|
||||
generatorsToRemove[index] = true;
|
||||
// console.log("\tGenerator ", index + 1, "has no more new ads");
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//Generator returned undefined, remove this generator from array
|
||||
generatorsToRemove[index] = true;
|
||||
// console.log("Generator ", index + 1, "has no more pages");
|
||||
}
|
||||
}
|
||||
|
||||
// console.log("Generators state : ", generatorsToRemove);
|
||||
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
|
||||
if (generatorsToRemove[i]) {
|
||||
// console.log("\tRemove generator ", i + 1);
|
||||
indexGenerators.splice(i, 1);
|
||||
}
|
||||
}
|
||||
if (indexGenerators.length === 0) {
|
||||
done = true;
|
||||
}
|
||||
|
||||
await this.sleep(this.delayBetweenPages);
|
||||
}
|
||||
}
|
||||
return newRealEstates;
|
||||
}
|
||||
|
||||
async *categoryIndexer(adCategory) {
|
||||
let pageToIndex = 1;
|
||||
|
||||
const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
|
||||
const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
|
||||
if (urlAdTypePart && urlCategoryPart) {
|
||||
while (true) {
|
||||
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
|
||||
const singlePageResults = await this.indexSinglePage(
|
||||
urlPageToCrawl,
|
||||
this.maxResultsPerPage
|
||||
);
|
||||
|
||||
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
|
||||
yield singlePageResults;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
++pageToIndex;
|
||||
if (pageToIndex === this.maxPages) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async indexSinglePage(url, maxResultsPerPage) {
|
||||
// console.log("[RENTAL] Index page : ", url);
|
||||
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
let hrefs = [];
|
||||
|
||||
$(
|
||||
"body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
|
||||
)
|
||||
.find(".pull-right")
|
||||
.each((i, elem) => {
|
||||
const href = $(elem)
|
||||
.find("a")
|
||||
.first()
|
||||
.attr("href");
|
||||
if (href) {
|
||||
hrefs.push(href);
|
||||
}
|
||||
});
|
||||
|
||||
let actualNoOfResults =
|
||||
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
|
||||
|
||||
const asyncScraping = [];
|
||||
for (let i = 0; i < actualNoOfResults; i++) {
|
||||
asyncScraping.push(this.scrapeAd(hrefs[i]));
|
||||
}
|
||||
|
||||
const scrapedData = await Promise.all(asyncScraping);
|
||||
const filteredScrapedData = scrapedData.filter(adData => !!adData);
|
||||
return filteredScrapedData;
|
||||
} catch (e) {
|
||||
console.error("[RENTAL] Exception caught:" + e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async scrapeAd(url) {
|
||||
// console.log("[RENTAL] Scraping : ", url);
|
||||
try {
|
||||
const adPageSource = await fetch(url);
|
||||
const body = await adPageSource.text();
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const mapElementParent = $(".box-map").parent();
|
||||
const scriptElement = $("script", mapElementParent);
|
||||
if (
|
||||
scriptElement[0] &&
|
||||
scriptElement[0].children &&
|
||||
scriptElement[0].children[0] &&
|
||||
scriptElement[0].children[0].data
|
||||
) {
|
||||
let extractedData;
|
||||
try {
|
||||
//data string starts with : var json_map_data = [{"r ...
|
||||
//so we remove first 20 characters
|
||||
|
||||
const jsonData = scriptElement[0].children[0].data.substring(20);
|
||||
const parsedJsonData = JSON.parse(jsonData);
|
||||
extractedData = parsedJsonData[0];
|
||||
} catch (e) {
|
||||
throw { message: "Can't find ad data JSON" };
|
||||
}
|
||||
|
||||
const rentalId = extractedData["re_realEstates_id"];
|
||||
const adCategory = this.getKiviCategoryIdFromRentalId(
|
||||
parseInt(extractedData["re_types_id"])
|
||||
);
|
||||
if (!adCategory) {
|
||||
throw {
|
||||
message: `Invalid category : ${extractedData["re_types_id"]}`
|
||||
};
|
||||
}
|
||||
const adType = this.getKiviAdTypeFromRentalActionId(
|
||||
parseInt(extractedData["re_action_id"])
|
||||
);
|
||||
if (!adType) {
|
||||
throw {
|
||||
message: `Invalid ad type : ${extractedData["re_action_id"]}`
|
||||
};
|
||||
}
|
||||
|
||||
const title = extractedData["re_realEstates_portalName"];
|
||||
const price = parseFloat(extractedData["re_realEstates_price"]);
|
||||
const area = parseFloat(extractedData["re_realEstates_area"]);
|
||||
const gardenSize = parseFloat(
|
||||
extractedData["re_realEstates_fieldArea"]
|
||||
);
|
||||
const longDescription = htmlToText.fromString(
|
||||
extractedData["re_realEstates_description"]
|
||||
);
|
||||
const locationLong = extractedData["re_realEstates_longitude"];
|
||||
const locationLat = extractedData["re_realEstates_latitude"];
|
||||
const publishedDateMoment = moment.tz(
|
||||
extractedData["re_realEstates_inserted"],
|
||||
RENTAL_ENUMS.RENTAL_PUBLISHED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
if (!publishedDateMoment.isValid()) {
|
||||
throw {
|
||||
message: `Invalid published date : ${
|
||||
extractedData["re_realEstates_inserted"]
|
||||
}`
|
||||
};
|
||||
}
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
extractedData["re_realEstates_edited"],
|
||||
RENTAL_ENUMS.RENTAL_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
if (!renewedDateMoment.isValid()) {
|
||||
throw {
|
||||
message: `Invalid renewed date : ${
|
||||
extractedData["re_realEstates_edited"]
|
||||
}`
|
||||
};
|
||||
}
|
||||
|
||||
const adStatus = AD_STATUS.STATUS_NORMAL;
|
||||
|
||||
const data = {
|
||||
url,
|
||||
agencyObjectId: rentalId,
|
||||
originAgencyName: AD_AGENCY.RENTAL,
|
||||
realEstateType: adCategory,
|
||||
adType,
|
||||
title,
|
||||
price,
|
||||
area,
|
||||
gardenSize,
|
||||
shortDescription: "",
|
||||
longDescription: longDescription,
|
||||
streetNumber: 0,
|
||||
streetName: "",
|
||||
locality: "",
|
||||
municipality: "",
|
||||
city: "",
|
||||
region: "",
|
||||
entity: "",
|
||||
country: "",
|
||||
locationLat,
|
||||
locationLong,
|
||||
adStatus,
|
||||
publishedDate: publishedDateMoment.toISOString(),
|
||||
renewedDate: renewedDateMoment.toISOString()
|
||||
};
|
||||
|
||||
return data;
|
||||
} else {
|
||||
console.log("[RENTAL] No JSON data for this ad : ", url);
|
||||
return null;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("[RENTAL] Exception caught: " + e.message, "\r\nURL:", url);
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//======= HELPER FUNCTIONS =============
|
||||
|
||||
getKiviCategoryIdFromRentalId(rentalCategoryId) {
|
||||
switch (rentalCategoryId) {
|
||||
case 1:
|
||||
return AD_CATEGORY.HOUSE.id;
|
||||
case 2:
|
||||
return AD_CATEGORY.FLAT.id;
|
||||
case 3:
|
||||
return AD_CATEGORY.APARTMENT.id;
|
||||
case 4:
|
||||
return AD_CATEGORY.OFFICE.id;
|
||||
case 5:
|
||||
return AD_CATEGORY.LAND.id;
|
||||
case 6:
|
||||
return AD_CATEGORY.GARAGE.id;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
getKiviAdTypeFromRentalActionId(actionId) {
|
||||
switch (actionId) {
|
||||
case 1:
|
||||
return AD_TYPE.AD_TYPE_SALE;
|
||||
case 2:
|
||||
return AD_TYPE.AD_TYPE_RENT;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async saveCrawledResults(results) {
|
||||
const savers = this.savers;
|
||||
|
||||
// for (const saver of savers) {
|
||||
// await saver.save(results);
|
||||
// }
|
||||
|
||||
//For now, we use only Postgres saver, so ...
|
||||
return await savers[0].save(results);
|
||||
//so that we can use some sequelize options and information when data is inserted
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RentalCrawler;
|
||||
Reference in New Issue
Block a user