move specific crawler config to the separated files
This commit is contained in:
Bilal Catic
565
app/crawler/specificCrawlers/olx.js
Normal file
565
app/crawler/specificCrawlers/olx.js
Normal file
@@ -0,0 +1,565 @@
|
||||
"use strict";
|
||||
|
||||
const fetch = require("node-fetch");
|
||||
const cheerio = require("cheerio");
|
||||
const Promise = require("bluebird");
|
||||
const moment = require("moment-timezone");
|
||||
|
||||
const {
|
||||
AD_TYPE,
|
||||
AD_CATEGORY,
|
||||
AD_AGENCY,
|
||||
AD_STATUS,
|
||||
CRAWLER_AD_TYPE
|
||||
} = require("../../common/enums");
|
||||
|
||||
const { DEFAULT_TIMEZONE } = require("../../config/appConfig");
|
||||
|
||||
const OLX_ENUMS = {
|
||||
OLX_AD_TYPE: {
|
||||
[CRAWLER_AD_TYPE.ALL]: "",
|
||||
[CRAWLER_AD_TYPE.ONLY_SELL]: "&vrsta=samoprodaja",
|
||||
[CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje"
|
||||
},
|
||||
OLX_AD_CATEGORY: {
|
||||
[AD_CATEGORY.FLAT.id]: "&kategorija=23",
|
||||
[AD_CATEGORY.HOUSE.id]: "&kategorija=24",
|
||||
[AD_CATEGORY.LAND.id]: "&kategorija=29",
|
||||
[AD_CATEGORY.OFFICE.id]: "&kategorija=25",
|
||||
[AD_CATEGORY.APARTMENT.id]: "&kategorija=27",
|
||||
[AD_CATEGORY.GARAGE.id]: "&kategorija=30",
|
||||
[AD_CATEGORY.COTTAGE.id]: "&kategorija=26"
|
||||
},
|
||||
MAX_DETAIL_FIELDS: 30,
|
||||
OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
|
||||
OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
|
||||
};
|
||||
|
||||
class OlxCrawler {
|
||||
constructor(
|
||||
savers = [],
|
||||
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
|
||||
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
|
||||
maxPages = 1000,
|
||||
maxResultsPerPage = 100,
|
||||
ignoredUsernames = [],
|
||||
delayBetweenPages = 1000
|
||||
) {
|
||||
this.savers = savers;
|
||||
this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
|
||||
this.crawlerAdTypes = crawlerAdTypes;
|
||||
this.crawlerAdCategories = crawlerAdCategories;
|
||||
this.maxPages = maxPages;
|
||||
this.maxResultsPerPage = maxResultsPerPage;
|
||||
this.ignoredUsernames = ignoredUsernames;
|
||||
this.delayBetweenPages = delayBetweenPages;
|
||||
}
|
||||
|
||||
async crawl() {
|
||||
const crawlAdCategories = this.crawlerAdCategories;
|
||||
|
||||
const newRealEstates = [];
|
||||
|
||||
if (crawlAdCategories) {
|
||||
const indexGenerators = [];
|
||||
for (const adCategory of crawlAdCategories) {
|
||||
indexGenerators.push(this.categoryIndexer(adCategory));
|
||||
}
|
||||
|
||||
let done = false;
|
||||
while (!done) {
|
||||
const categoryIndexerPromises = [];
|
||||
const generatorsToRemove = [];
|
||||
for (const indexGenerator of indexGenerators) {
|
||||
categoryIndexerPromises.push(indexGenerator.next());
|
||||
generatorsToRemove.push(false);
|
||||
}
|
||||
|
||||
const singlePageResults = await Promise.all(categoryIndexerPromises);
|
||||
const entries = singlePageResults.entries();
|
||||
|
||||
for (const [index, { value: singlePageResult }] of entries) {
|
||||
if (singlePageResult) {
|
||||
const saveResults = await this.saveCrawledResults(singlePageResult);
|
||||
const { newRecords, existingRecords } = saveResults;
|
||||
|
||||
newRealEstates.push(...newRecords);
|
||||
|
||||
for (const existingRecord of existingRecords) {
|
||||
const { publishedDate, renewedDate } = existingRecord;
|
||||
|
||||
const publishedDateMoment = moment.utc(publishedDate);
|
||||
const renewedDateMoment = moment.utc(renewedDate);
|
||||
|
||||
const stopCrawlingThisCategory = publishedDateMoment.isSame(
|
||||
renewedDateMoment,
|
||||
"minute"
|
||||
);
|
||||
|
||||
if (stopCrawlingThisCategory) {
|
||||
generatorsToRemove[index] = true;
|
||||
// console.log("\tGenerator ", index + 1, "has no more new ads");
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//Generator returned undefined, remove this generator from array
|
||||
generatorsToRemove[index] = true;
|
||||
// console.log("Generator ", index + 1, "has no more pages");
|
||||
}
|
||||
}
|
||||
|
||||
// console.log("Generators state : ", generatorsToRemove);
|
||||
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
|
||||
if (generatorsToRemove[i]) {
|
||||
// console.log("\tRemove generator ", i + 1);
|
||||
indexGenerators.splice(i, 1);
|
||||
}
|
||||
}
|
||||
if (indexGenerators.length === 0) {
|
||||
done = true;
|
||||
}
|
||||
|
||||
await this.sleep(this.delayBetweenPages);
|
||||
}
|
||||
}
|
||||
return newRealEstates;
|
||||
}
|
||||
|
||||
async *categoryIndexer(adCategory) {
|
||||
let pageToIndex = 1;
|
||||
|
||||
const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
|
||||
const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
|
||||
if (urlAdTypePart && urlCategoryPart) {
|
||||
while (true) {
|
||||
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
|
||||
const singlePageResults = await this.indexSinglePage(
|
||||
urlPageToCrawl,
|
||||
this.maxResultsPerPage
|
||||
);
|
||||
|
||||
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
|
||||
yield singlePageResults;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
++pageToIndex;
|
||||
if (pageToIndex === this.maxPages) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async indexSinglePage(url, maxResultsPerPage) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
let hrefs = [];
|
||||
|
||||
$("#rezultatipretrage")
|
||||
.find(".listitem")
|
||||
.each((i, elem) => {
|
||||
const href = $(elem)
|
||||
.find("a")
|
||||
.first()
|
||||
.attr("href");
|
||||
if (href) {
|
||||
hrefs.push(href);
|
||||
}
|
||||
});
|
||||
|
||||
let actualNoOfResults =
|
||||
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
|
||||
|
||||
const asyncScraping = [];
|
||||
for (let i = 0; i < actualNoOfResults; i++) {
|
||||
asyncScraping.push(this.scrapeAd(hrefs[i]));
|
||||
}
|
||||
|
||||
const scrapedData = await Promise.all(asyncScraping);
|
||||
const filteredScrapedData = scrapedData.filter(adData => !!adData);
|
||||
return filteredScrapedData;
|
||||
} catch (e) {
|
||||
console.error("Exception caught:" + e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async scrapeAd(url) {
|
||||
// console.log("Scraping : ", url);
|
||||
try {
|
||||
const adPageSource = await fetch(url);
|
||||
const body = await adPageSource.text();
|
||||
const $ = cheerio.load(body);
|
||||
let status = AD_STATUS.STATUS_NORMAL;
|
||||
|
||||
const propertySelectors = {
|
||||
username:
|
||||
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
|
||||
title: "#naslovartikla",
|
||||
descriptions: ".artikal_detaljniopis_tekst",
|
||||
category:
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span"
|
||||
};
|
||||
|
||||
const username = $(propertySelectors.username)
|
||||
.text()
|
||||
.trim();
|
||||
if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const title = $(propertySelectors.title)
|
||||
.text()
|
||||
.trim();
|
||||
const descriptions = $(propertySelectors.descriptions);
|
||||
const category = $(propertySelectors.category)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//====== PRICE DETECTION AND EXTRACTION =====
|
||||
let price = null;
|
||||
const normalPriceValue = $("#pc > p:nth-child(2)").text();
|
||||
const urgentPriceValue = $(
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (normalPriceValue && normalPriceValue.length > 0) {
|
||||
price = normalPriceValue;
|
||||
if (
|
||||
$("#pc > p.n")
|
||||
.text()
|
||||
.indexOf("Hitna") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
status = AD_STATUS.STATUS_NORMAL;
|
||||
}
|
||||
} else if (urgentPriceValue && urgentPriceValue.length > 0) {
|
||||
const priceValues = urgentPriceValue.split("KM");
|
||||
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
|
||||
if (priceValues.length > 1) {
|
||||
price = priceValues[1].trim();
|
||||
status = AD_STATUS.STATUS_DISCOUNTED;
|
||||
} else {
|
||||
throw { message: "Can't find urgent price" };
|
||||
}
|
||||
} else {
|
||||
throw {
|
||||
message: "Can't find price (it is not normal nor urgent price ?)"
|
||||
};
|
||||
}
|
||||
|
||||
//====== OTHER AD INFORMATION ===============
|
||||
let adType = null;
|
||||
let olxId = null;
|
||||
|
||||
let otherInformationDivId;
|
||||
//We need to locate DIV ID where other information are stored
|
||||
for (let possibleId = 10; possibleId <= 20; possibleId++) {
|
||||
const adTypeFieldTitle = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (adTypeFieldTitle === "Vrsta oglasa") {
|
||||
otherInformationDivId = possibleId;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!otherInformationDivId) {
|
||||
throw { message: "Other information DIV could not be found" };
|
||||
}
|
||||
|
||||
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
|
||||
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
|
||||
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
|
||||
|
||||
const publishedDate = $(publishedDateValueSelector)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const publishedDateMoment = moment.tz(
|
||||
publishedDate,
|
||||
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!publishedDateMoment.isValid()) {
|
||||
throw { message: "Invalid published date ! Check parsing format" };
|
||||
}
|
||||
|
||||
const renewedDate = $(renewedDateFullValueSelector)
|
||||
.data("content")
|
||||
.trim();
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
renewedDate,
|
||||
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!renewedDateMoment) {
|
||||
throw {
|
||||
message:
|
||||
"Invalid renewed date ! Check how parser parsed renewed date text"
|
||||
};
|
||||
}
|
||||
|
||||
adType = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
|
||||
.text()
|
||||
.trim();
|
||||
olxId = $(`${olxIdFieldSelector} > div.df2`)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (olxIdFieldTitle !== "OLX ID") {
|
||||
throw { message: "Cannot find correct OLX ID" };
|
||||
}
|
||||
//===========================================
|
||||
|
||||
//====== DETAIL INFORMATION FIELDS ==========
|
||||
let area = null;
|
||||
let gardenSize = null;
|
||||
|
||||
let fieldIndex = 1;
|
||||
do {
|
||||
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
|
||||
const fieldTitleSelector = `${fieldSelector} > div.df1`;
|
||||
const fieldValueSelector = `${fieldSelector} > div.df2`;
|
||||
|
||||
const fieldTitle = $(fieldTitleSelector)
|
||||
.text()
|
||||
.trim();
|
||||
const fieldValue = $(fieldValueSelector)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
switch (fieldTitle) {
|
||||
case "Kvadrata":
|
||||
area = fieldValue;
|
||||
break;
|
||||
case "Okućnica (kvadratura)":
|
||||
gardenSize = fieldValue;
|
||||
break;
|
||||
}
|
||||
|
||||
if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
//===========================================
|
||||
|
||||
//====== UNUSED FIELDS FOR NOW ==============
|
||||
const time = $("time").attr("datetime");
|
||||
const numberOfViews = $(
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(18) > div:nth-child(6) > div.df2"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
//===========================================
|
||||
|
||||
//=========================================
|
||||
const parsedCategory = this.getAdCategoryId(category);
|
||||
if (!parsedCategory) {
|
||||
throw { message: "Unknown ad category" };
|
||||
}
|
||||
|
||||
const parsedAdType = this.getAdTypeId(adType);
|
||||
if (!parsedAdType) {
|
||||
throw { message: "Unknown ad type" };
|
||||
}
|
||||
|
||||
const parsedArea = this.parseArea(area) || null;
|
||||
const parsedGardenSize = this.parseArea(gardenSize) || null;
|
||||
const parsedPrice = this.parsePrice(price) || null;
|
||||
|
||||
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
||||
const locationLatLngMatches = latLngRegex.exec(body);
|
||||
|
||||
let locationLat = null;
|
||||
let locationLong = null;
|
||||
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
|
||||
locationLat = parseFloat(locationLatLngMatches[1]) || null;
|
||||
locationLong = parseFloat(locationLatLngMatches[2]) || null;
|
||||
}
|
||||
|
||||
const data = {
|
||||
url,
|
||||
agencyObjectId: olxId,
|
||||
originAgencyName: AD_AGENCY.OLX,
|
||||
realEstateType: parsedCategory,
|
||||
adType: parsedAdType,
|
||||
title,
|
||||
price: parsedPrice,
|
||||
area: parsedArea,
|
||||
gardenSize: parsedGardenSize,
|
||||
shortDescription: descriptions
|
||||
.first()
|
||||
.text()
|
||||
.trim(),
|
||||
longDescription: descriptions
|
||||
.last()
|
||||
.text()
|
||||
.trim(),
|
||||
streetNumber: 0,
|
||||
streetName: "",
|
||||
locality: "",
|
||||
municipality: "",
|
||||
city: "",
|
||||
region: "",
|
||||
entity: "",
|
||||
country: "",
|
||||
locationLat,
|
||||
locationLong,
|
||||
adStatus: status,
|
||||
publishedDate: publishedDateMoment.toISOString(),
|
||||
renewedDate: renewedDateMoment.toISOString()
|
||||
};
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
console.error("Exception caught: " + e.message, "\r\nURL:", url);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//======= HELPER FUNCTIONS =============
|
||||
|
||||
getAdCategoryId(categoryText) {
|
||||
switch (categoryText) {
|
||||
case "Stanovi":
|
||||
return AD_CATEGORY.FLAT.id;
|
||||
case "Zemljišta":
|
||||
return AD_CATEGORY.LAND.id;
|
||||
case "Kuće":
|
||||
return AD_CATEGORY.HOUSE.id;
|
||||
case "Poslovni prostori":
|
||||
return AD_CATEGORY.OFFICE.id;
|
||||
case "Apartmani":
|
||||
return AD_CATEGORY.APARTMENT.id;
|
||||
case "Garaže":
|
||||
return AD_CATEGORY.GARAGE.id;
|
||||
case "Vikendice":
|
||||
return AD_CATEGORY.COTTAGE.id;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
getAdTypeId(adTypeText) {
|
||||
switch (adTypeText) {
|
||||
case "Prodaja":
|
||||
return AD_TYPE.AD_TYPE_SALE;
|
||||
case "Izdavanje":
|
||||
return AD_TYPE.AD_TYPE_RENT;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
parseArea(areaText) {
|
||||
if (!areaText) {
|
||||
return NaN;
|
||||
}
|
||||
const removeDotsExceptLastOneRegex = /[.](?=.*[.])/g;
|
||||
const textWithOnlyOneDecimalDot = areaText
|
||||
.replace(",", ".")
|
||||
.replace(removeDotsExceptLastOneRegex, "");
|
||||
|
||||
return parseFloat(textWithOnlyOneDecimalDot);
|
||||
}
|
||||
|
||||
parsePrice(priceText) {
|
||||
if (!priceText) {
|
||||
return NaN;
|
||||
}
|
||||
const formattedPriceText = priceText.replace(".", "").replace(",", ".");
|
||||
return parseFloat(formattedPriceText);
|
||||
}
|
||||
|
||||
parseRenewedDate(renewedDateText) {
|
||||
const currentMoment = moment.tz(DEFAULT_TIMEZONE);
|
||||
|
||||
if (renewedDateText.includes("Prije mjesec dana")) {
|
||||
return currentMoment.add(-1, "month");
|
||||
}
|
||||
|
||||
if (renewedDateText.includes("Jučer")) {
|
||||
return currentMoment.add(-1, "day");
|
||||
}
|
||||
|
||||
if (renewedDateText.includes("Prije sat")) {
|
||||
return currentMoment.add(-1, "hour");
|
||||
}
|
||||
|
||||
if (renewedDateText.includes("dan")) {
|
||||
// format for this case should be "Prije N dana" or "Prije N dan"
|
||||
const dateParts = renewedDateText.split(" ");
|
||||
if (dateParts[0] === "Prije") {
|
||||
const numberOfDays = parseInt(dateParts[1]);
|
||||
return currentMoment.add(-1 * numberOfDays, "days");
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if (renewedDateText.includes("sat")) {
|
||||
const dateParts = renewedDateText.split(" ");
|
||||
const parsedHours =
|
||||
dateParts && dateParts.length > 2 ? parseInt(dateParts[1]) : undefined;
|
||||
if (!parsedHours) {
|
||||
return undefined;
|
||||
}
|
||||
return currentMoment.add(-1 * parsedHours, "hours");
|
||||
}
|
||||
|
||||
const todayVariations = ["min", "sekund", "maloprije"];
|
||||
for (const todayVariation of todayVariations) {
|
||||
if (renewedDateText.includes(todayVariation)) {
|
||||
return currentMoment;
|
||||
}
|
||||
}
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
renewedDateText,
|
||||
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
return renewedDateMoment.isValid() ? renewedDateMoment : undefined;
|
||||
}
|
||||
|
||||
async sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async saveCrawledResults(results) {
|
||||
const savers = this.savers;
|
||||
|
||||
// for (const saver of savers) {
|
||||
// await saver.save(results);
|
||||
// }
|
||||
|
||||
//For now, we use only Postgres saver, so ...
|
||||
return await savers[0].save(results);
|
||||
//so that we can use some sequelize options and information when data is inserted
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = OlxCrawler;
|
||||
376
app/crawler/specificCrawlers/rental.js
Normal file
376
app/crawler/specificCrawlers/rental.js
Normal file
@@ -0,0 +1,376 @@
|
||||
"use strict";
|
||||
|
||||
const fetch = require("node-fetch");
|
||||
const cheerio = require("cheerio");
|
||||
const Promise = require("bluebird");
|
||||
const moment = require("moment-timezone");
|
||||
const htmlToText = require("html-to-text");
|
||||
|
||||
const {
|
||||
AD_TYPE,
|
||||
AD_CATEGORY,
|
||||
AD_AGENCY,
|
||||
AD_STATUS,
|
||||
CRAWLER_AD_TYPE
|
||||
} = require("../../common/enums");
|
||||
|
||||
const { DEFAULT_TIMEZONE } = require("../../config/appConfig");
|
||||
|
||||
const RENTAL_ENUMS = {
|
||||
RENTAL_AD_TYPE: {
|
||||
[CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
|
||||
[CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
|
||||
[CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
|
||||
},
|
||||
RENTAL_AD_CATEGORY: {
|
||||
[AD_CATEGORY.FLAT.id]: "/tip-2",
|
||||
[AD_CATEGORY.HOUSE.id]: "/tip-1",
|
||||
[AD_CATEGORY.LAND.id]: "/tip-5",
|
||||
[AD_CATEGORY.OFFICE.id]: "/tip-4",
|
||||
[AD_CATEGORY.APARTMENT.id]: "/tip-3",
|
||||
[AD_CATEGORY.GARAGE.id]: "/tip-6"
|
||||
//[AD_CATEGORY.COTTAGE.id]: ""
|
||||
},
|
||||
RENTAL_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
|
||||
RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
|
||||
};
|
||||
|
||||
class RentalCrawler {
|
||||
constructor(
|
||||
savers = [],
|
||||
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
|
||||
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
|
||||
maxPages = 1000,
|
||||
maxResultsPerPage = 100,
|
||||
ignoredUsernames = [],
|
||||
delayBetweenPages = 1000
|
||||
) {
|
||||
this.savers = savers;
|
||||
this.baseUrl = "https://www.rental.ba/pretraga/sortiraj-date_DESC";
|
||||
this.crawlerAdTypes = crawlerAdTypes;
|
||||
this.crawlerAdCategories = crawlerAdCategories;
|
||||
this.maxPages = maxPages;
|
||||
this.maxResultsPerPage = maxResultsPerPage;
|
||||
this.delayBetweenPages = delayBetweenPages;
|
||||
}
|
||||
|
||||
async crawl() {
|
||||
const crawlAdCategories = this.crawlerAdCategories;
|
||||
|
||||
const newRealEstates = [];
|
||||
|
||||
if (crawlAdCategories) {
|
||||
const indexGenerators = [];
|
||||
for (const adCategory of crawlAdCategories) {
|
||||
indexGenerators.push(this.categoryIndexer(adCategory));
|
||||
}
|
||||
|
||||
let done = false;
|
||||
while (!done) {
|
||||
const categoryIndexerPromises = [];
|
||||
const generatorsToRemove = [];
|
||||
for (const indexGenerator of indexGenerators) {
|
||||
categoryIndexerPromises.push(indexGenerator.next());
|
||||
generatorsToRemove.push(false);
|
||||
}
|
||||
|
||||
const singlePageResults = await Promise.all(categoryIndexerPromises);
|
||||
const entries = singlePageResults.entries();
|
||||
|
||||
for (const [index, { value: singlePageResult }] of entries) {
|
||||
if (singlePageResult) {
|
||||
const saveResults = await this.saveCrawledResults(singlePageResult);
|
||||
const { newRecords } = saveResults;
|
||||
|
||||
newRealEstates.push(...newRecords);
|
||||
|
||||
if (Array.isArray(newRecords) && newRecords.length === 0) {
|
||||
generatorsToRemove[index] = true;
|
||||
}
|
||||
|
||||
// for (const existingRecord of existingRecords) {
|
||||
// const { publishedDate, renewedDate } = existingRecord;
|
||||
//
|
||||
// const publishedDateMoment = moment.utc(publishedDate);
|
||||
// const renewedDateMoment = moment.utc(renewedDate);
|
||||
//
|
||||
// const stopCrawlingThisCategory = publishedDateMoment.isSame(
|
||||
// renewedDateMoment,
|
||||
// "minute"
|
||||
// );
|
||||
//
|
||||
// if (stopCrawlingThisCategory) {
|
||||
// generatorsToRemove[index] = true;
|
||||
// // console.log("\tGenerator ", index + 1, "has no more new ads");
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
} else {
|
||||
//Generator returned undefined, remove this generator from array
|
||||
generatorsToRemove[index] = true;
|
||||
// console.log("Generator ", index + 1, "has no more pages");
|
||||
}
|
||||
}
|
||||
|
||||
// console.log("Generators state : ", generatorsToRemove);
|
||||
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
|
||||
if (generatorsToRemove[i]) {
|
||||
// console.log("\tRemove generator ", i + 1);
|
||||
indexGenerators.splice(i, 1);
|
||||
}
|
||||
}
|
||||
if (indexGenerators.length === 0) {
|
||||
done = true;
|
||||
}
|
||||
|
||||
await this.sleep(this.delayBetweenPages);
|
||||
}
|
||||
}
|
||||
return newRealEstates;
|
||||
}
|
||||
|
||||
async *categoryIndexer(adCategory) {
|
||||
let pageToIndex = 1;
|
||||
|
||||
const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
|
||||
const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
|
||||
if (urlAdTypePart && urlCategoryPart) {
|
||||
while (true) {
|
||||
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
|
||||
const singlePageResults = await this.indexSinglePage(
|
||||
urlPageToCrawl,
|
||||
this.maxResultsPerPage
|
||||
);
|
||||
|
||||
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
|
||||
yield singlePageResults;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
++pageToIndex;
|
||||
if (pageToIndex === this.maxPages) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async indexSinglePage(url, maxResultsPerPage) {
|
||||
// console.log("[RENTAL] Index page : ", url);
|
||||
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
let hrefs = [];
|
||||
|
||||
$(
|
||||
"body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
|
||||
)
|
||||
.find(".pull-right")
|
||||
.each((i, elem) => {
|
||||
const href = $(elem)
|
||||
.find("a")
|
||||
.first()
|
||||
.attr("href");
|
||||
if (href) {
|
||||
hrefs.push(href);
|
||||
}
|
||||
});
|
||||
|
||||
let actualNoOfResults =
|
||||
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
|
||||
|
||||
const asyncScraping = [];
|
||||
for (let i = 0; i < actualNoOfResults; i++) {
|
||||
asyncScraping.push(this.scrapeAd(hrefs[i]));
|
||||
}
|
||||
|
||||
const scrapedData = await Promise.all(asyncScraping);
|
||||
const filteredScrapedData = scrapedData.filter(adData => !!adData);
|
||||
return filteredScrapedData;
|
||||
} catch (e) {
|
||||
console.error("[RENTAL] Exception caught:" + e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async scrapeAd(url) {
|
||||
// console.log("[RENTAL] Scraping : ", url);
|
||||
try {
|
||||
const adPageSource = await fetch(url);
|
||||
const body = await adPageSource.text();
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const mapElementParent = $(".box-map").parent();
|
||||
const scriptElement = $("script", mapElementParent);
|
||||
if (
|
||||
scriptElement[0] &&
|
||||
scriptElement[0].children &&
|
||||
scriptElement[0].children[0] &&
|
||||
scriptElement[0].children[0].data
|
||||
) {
|
||||
let extractedData;
|
||||
try {
|
||||
//data string starts with : var json_map_data = [{"r ...
|
||||
//so we remove first 20 characters
|
||||
|
||||
const jsonData = scriptElement[0].children[0].data.substring(20);
|
||||
const parsedJsonData = JSON.parse(jsonData);
|
||||
extractedData = parsedJsonData[0];
|
||||
} catch (e) {
|
||||
throw { message: "Can't find ad data JSON" };
|
||||
}
|
||||
|
||||
const rentalId = extractedData["re_realEstates_id"];
|
||||
const adCategory = this.getKiviCategoryIdFromRentalId(
|
||||
parseInt(extractedData["re_types_id"])
|
||||
);
|
||||
if (!adCategory) {
|
||||
throw {
|
||||
message: `Invalid category : ${extractedData["re_types_id"]}`
|
||||
};
|
||||
}
|
||||
const adType = this.getKiviAdTypeFromRentalActionId(
|
||||
parseInt(extractedData["re_action_id"])
|
||||
);
|
||||
if (!adType) {
|
||||
throw {
|
||||
message: `Invalid ad type : ${extractedData["re_action_id"]}`
|
||||
};
|
||||
}
|
||||
|
||||
const title = extractedData["re_realEstates_portalName"];
|
||||
const extractedPrice = parseFloat(
|
||||
extractedData["re_realEstates_price"]
|
||||
);
|
||||
const price = extractedPrice ? extractedPrice : null;
|
||||
const area = parseFloat(extractedData["re_realEstates_area"]);
|
||||
const gardenSize = parseFloat(
|
||||
extractedData["re_realEstates_fieldArea"]
|
||||
);
|
||||
const longDescription = htmlToText.fromString(
|
||||
extractedData["re_realEstates_description"]
|
||||
);
|
||||
const locationLong = extractedData["re_realEstates_longitude"];
|
||||
const locationLat = extractedData["re_realEstates_latitude"];
|
||||
const publishedDateMoment = moment.tz(
|
||||
extractedData["re_realEstates_inserted"],
|
||||
RENTAL_ENUMS.RENTAL_PUBLISHED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
if (!publishedDateMoment.isValid()) {
|
||||
throw {
|
||||
message: `Invalid published date : ${
|
||||
extractedData["re_realEstates_inserted"]
|
||||
}`
|
||||
};
|
||||
}
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
extractedData["re_realEstates_edited"],
|
||||
RENTAL_ENUMS.RENTAL_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
if (!renewedDateMoment.isValid()) {
|
||||
throw {
|
||||
message: `Invalid renewed date : ${
|
||||
extractedData["re_realEstates_edited"]
|
||||
}`
|
||||
};
|
||||
}
|
||||
|
||||
const adStatus = AD_STATUS.STATUS_NORMAL;
|
||||
|
||||
const data = {
|
||||
url,
|
||||
agencyObjectId: rentalId,
|
||||
originAgencyName: AD_AGENCY.RENTAL,
|
||||
realEstateType: adCategory,
|
||||
adType,
|
||||
title,
|
||||
price,
|
||||
area,
|
||||
gardenSize,
|
||||
shortDescription: "",
|
||||
longDescription: longDescription,
|
||||
streetNumber: 0,
|
||||
streetName: "",
|
||||
locality: "",
|
||||
municipality: "",
|
||||
city: "",
|
||||
region: "",
|
||||
entity: "",
|
||||
country: "",
|
||||
locationLat,
|
||||
locationLong,
|
||||
adStatus,
|
||||
publishedDate: publishedDateMoment.toISOString(),
|
||||
renewedDate: renewedDateMoment.toISOString()
|
||||
};
|
||||
|
||||
return data;
|
||||
} else {
|
||||
console.log("[RENTAL] No JSON data for this ad : ", url);
|
||||
return null;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("[RENTAL] Exception caught: " + e.message, "\r\nURL:", url);
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//======= HELPER FUNCTIONS =============
|
||||
|
||||
getKiviCategoryIdFromRentalId(rentalCategoryId) {
|
||||
switch (rentalCategoryId) {
|
||||
case 1:
|
||||
return AD_CATEGORY.HOUSE.id;
|
||||
case 2:
|
||||
return AD_CATEGORY.FLAT.id;
|
||||
case 3:
|
||||
return AD_CATEGORY.APARTMENT.id;
|
||||
case 4:
|
||||
return AD_CATEGORY.OFFICE.id;
|
||||
case 5:
|
||||
return AD_CATEGORY.LAND.id;
|
||||
case 6:
|
||||
return AD_CATEGORY.GARAGE.id;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
getKiviAdTypeFromRentalActionId(actionId) {
|
||||
switch (actionId) {
|
||||
case 1:
|
||||
return AD_TYPE.AD_TYPE_SALE;
|
||||
case 2:
|
||||
return AD_TYPE.AD_TYPE_RENT;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async saveCrawledResults(results) {
|
||||
const savers = this.savers;
|
||||
|
||||
// for (const saver of savers) {
|
||||
// await saver.save(results);
|
||||
// }
|
||||
|
||||
//For now, we use only Postgres saver, so ...
|
||||
return await savers[0].save(results);
|
||||
//so that we can use some sequelize options and information when data is inserted
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RentalCrawler;
|
||||
Reference in New Issue
Block a user