387 lines
11 KiB
JavaScript
387 lines
11 KiB
JavaScript
"use strict";
|
|
|
|
let fetch = require("node-fetch");
|
|
let cheerio = require("cheerio");
|
|
|
|
const {
|
|
AD_TYPE,
|
|
AD_CATEGORY,
|
|
IGNORED_USERNAMES,
|
|
AD_AGENCY,
|
|
AD_STATUS,
|
|
CRAWLER_AD_TYPE
|
|
} = require("../../common/enums");
|
|
|
|
const OLX_ENUMS = {
|
|
OLX_AD_TYPE: {},
|
|
OLX_AD_CATEGORY: {},
|
|
MAX_DETAIL_FIELDS: 30
|
|
};
|
|
|
|
OLX_ENUMS.OLX_AD_TYPE[CRAWLER_AD_TYPE.ALL] = "";
|
|
OLX_ENUMS.OLX_AD_TYPE[CRAWLER_AD_TYPE.ONLY_SELL] = "&vrsta=samoprodaja";
|
|
OLX_ENUMS.OLX_AD_TYPE[CRAWLER_AD_TYPE.ONLY_RENT] = "&vrsta=samoizdavanje";
|
|
|
|
OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_FLAT] = "&kategorija=23";
|
|
OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_HOUSE] = "&kategorija=24";
|
|
OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_LAND] = "&kategorija=29";
|
|
OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_OFFICE] = "&kategorija=25";
|
|
OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_APARTMENT] = "&kategorija=27";
|
|
OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_GARAGE] = "&kategorija=30";
|
|
|
|
class OlxCrawler {
|
|
constructor(
|
|
fromPage = 1,
|
|
toPage = 10,
|
|
maxResults = 1000,
|
|
savers = [],
|
|
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
|
|
crawlerAdCategories = [
|
|
AD_CATEGORY.CATEGORY_FLAT,
|
|
AD_CATEGORY.CATEGORY_HOUSE
|
|
]
|
|
) {
|
|
this.fromPage = fromPage;
|
|
this.toPage = toPage;
|
|
this.maxResults = maxResults;
|
|
this.savers = savers;
|
|
this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
|
|
this.crawlerAdTypes = crawlerAdTypes;
|
|
this.crawlerAdCategories = crawlerAdCategories;
|
|
}
|
|
|
|
async crawl() {
|
|
console.log("[OLX] Crawler started");
|
|
const crawlAdTypes = this.crawlerAdTypes;
|
|
const crawlAdCategories = this.crawlerAdCategories;
|
|
|
|
const urlWithAdTypeFilter = `${this.baseUrl}${OLX_ENUMS.OLX_AD_TYPE[crawlAdTypes]}`;
|
|
|
|
if (crawlAdCategories && crawlAdTypes) {
|
|
const asyncPagesIndexingByCategory = [];
|
|
for (const adCategory of crawlAdCategories) {
|
|
asyncPagesIndexingByCategory.push(
|
|
this.indexPages(
|
|
`${urlWithAdTypeFilter}${OLX_ENUMS.OLX_AD_CATEGORY[adCategory]}`
|
|
)
|
|
);
|
|
}
|
|
|
|
await Promise.all(asyncPagesIndexingByCategory);
|
|
}
|
|
console.log("[OLX] Crawler finished");
|
|
}
|
|
|
|
async indexPages(url) {
|
|
const startPage = this.fromPage;
|
|
const endPage = this.toPage;
|
|
const maxResultsPerPage = this.maxResults;
|
|
|
|
for (let pageNumber = startPage; pageNumber <= endPage; pageNumber++) {
|
|
const singlePageResults = await this.indexSinglePage(
|
|
url,
|
|
pageNumber,
|
|
maxResultsPerPage
|
|
);
|
|
await this.saveCrawledResults(singlePageResults);
|
|
await this.sleep(5000);
|
|
}
|
|
}
|
|
|
|
async indexSinglePage(urlWithoutPageNumber, pageNumber, maxResultsPerPage) {
|
|
try {
|
|
const url = `${urlWithoutPageNumber}&stranica=${pageNumber}`;
|
|
|
|
const res = await fetch(url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
let hrefs = [];
|
|
const singlePageResults = [];
|
|
|
|
$("#rezultatipretrage")
|
|
.find(".listitem")
|
|
.each((i, elem) => {
|
|
const href = $(elem)
|
|
.find("a")
|
|
.first()
|
|
.attr("href");
|
|
if (href) {
|
|
hrefs.push(href);
|
|
}
|
|
});
|
|
|
|
let actualNoOfResults =
|
|
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
|
|
|
|
for (let i = 0; i < actualNoOfResults; i++) {
|
|
console.log(`Scraping : ${hrefs[i]}`);
|
|
|
|
const adData = await this.scrapeAd(hrefs[i]);
|
|
|
|
if (adData) {
|
|
singlePageResults.push(adData);
|
|
}
|
|
await this.sleep(500);
|
|
}
|
|
|
|
return singlePageResults;
|
|
} catch (e) {
|
|
console.error("Exception caught:" + e);
|
|
}
|
|
}
|
|
|
|
async scrapeAd(url) {
|
|
try {
|
|
const adPageSource = await fetch(url);
|
|
const body = await adPageSource.text();
|
|
const $ = cheerio.load(body);
|
|
let status = AD_STATUS.STATUS_NORMAL;
|
|
|
|
const username = $(
|
|
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span"
|
|
).text();
|
|
|
|
if (IGNORED_USERNAMES.includes((username || "").toLowerCase())) {
|
|
return null;
|
|
}
|
|
|
|
const title = $("#naslovartikla").text();
|
|
const descriptions = $(".artikal_detaljniopis_tekst");
|
|
const category = $(
|
|
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span"
|
|
).text();
|
|
|
|
//====== PRICE DETECTION AND EXTRACTION =====
|
|
let price = null;
|
|
const normalPriceValue = $("#pc > p:nth-child(2)").text();
|
|
const urgentPriceValue = $(
|
|
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
|
|
).text();
|
|
|
|
if (normalPriceValue && normalPriceValue.length > 0) {
|
|
price = normalPriceValue;
|
|
if (
|
|
$("#pc > p.n")
|
|
.text()
|
|
.indexOf("Hitna") !== -1
|
|
) {
|
|
status = AD_STATUS.STATUS_URGENT;
|
|
} else {
|
|
status = AD_STATUS.STATUS_NORMAL;
|
|
}
|
|
} else if (urgentPriceValue && urgentPriceValue.length > 0) {
|
|
const priceValues = urgentPriceValue.split("KM");
|
|
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
|
|
if (priceValues.length > 1) {
|
|
price = priceValues[1].trim();
|
|
status = AD_STATUS.STATUS_DISCOUNTED;
|
|
} else {
|
|
throw { message: "Can't find urgent price" };
|
|
}
|
|
} else {
|
|
throw {
|
|
message: "Can't find price (it is not normal nor urgent price ?)"
|
|
};
|
|
}
|
|
|
|
//====== OTHER AD INFORMATION ===============
|
|
let adType = null;
|
|
let olxId = null;
|
|
|
|
let otherInformationDivId;
|
|
//We need to locate DIV ID where other information are stored
|
|
for (let possibleId = 10; possibleId <= 20; possibleId++) {
|
|
const adTypeFieldTitle = $(
|
|
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
|
|
)
|
|
.text()
|
|
.trim();
|
|
|
|
if (adTypeFieldTitle === "Vrsta oglasa") {
|
|
otherInformationDivId = possibleId;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!otherInformationDivId) {
|
|
throw { message: "Other information DIV could not be found" };
|
|
}
|
|
|
|
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
|
|
|
|
adType = $(
|
|
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
|
|
)
|
|
.text()
|
|
.trim();
|
|
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
|
|
.text()
|
|
.trim();
|
|
olxId = $(`${olxIdFieldSelector} > div.df2`)
|
|
.text()
|
|
.trim();
|
|
|
|
if (olxIdFieldTitle !== "OLX ID") {
|
|
throw { message: "Cannot find correct OLX ID" };
|
|
}
|
|
//===========================================
|
|
|
|
//====== DETAIL INFORMATION FIELDS ==========
|
|
let area = null;
|
|
let gardenSize = null;
|
|
|
|
let fieldIndex = 1;
|
|
do {
|
|
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
|
|
const fieldTitleSelector = `${fieldSelector} > div.df1`;
|
|
const fieldValueSelector = `${fieldSelector} > div.df2`;
|
|
|
|
const fieldTitle = $(fieldTitleSelector)
|
|
.text()
|
|
.trim();
|
|
const fieldValue = $(fieldValueSelector)
|
|
.text()
|
|
.trim();
|
|
|
|
switch (fieldTitle) {
|
|
case "Kvadrata":
|
|
area = fieldValue;
|
|
break;
|
|
case "Okućnica (kvadratura)":
|
|
gardenSize = fieldValue;
|
|
break;
|
|
}
|
|
|
|
if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
|
|
break;
|
|
}
|
|
} while (true);
|
|
//===========================================
|
|
|
|
//====== UNUSED FIELDS FOR NOW ==============
|
|
const time = $("time").attr("datetime");
|
|
const numberOfViews = $(
|
|
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(18) > div:nth-child(6) > div.df2"
|
|
).text();
|
|
//===========================================
|
|
|
|
//=========================================
|
|
const parsedCategory = this.getAdCategoryId(category);
|
|
if (!parsedCategory) {
|
|
throw { message: "Unknown ad category" };
|
|
}
|
|
|
|
const parsedAdType = this.getAdTypeId(adType);
|
|
if (!parsedAdType) {
|
|
throw { message: "Unknown ad type" };
|
|
}
|
|
|
|
const parsedArea = this.parseArea(area) || null;
|
|
const parsedGardenSize = this.parseArea(gardenSize) || null;
|
|
const parsedPrice = this.parsePrice(price) || null;
|
|
|
|
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
|
const locationLatLngMatches = latLngRegex.exec(body);
|
|
|
|
let locationLat = null;
|
|
let locationLong = null;
|
|
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
|
|
locationLat = parseFloat(locationLatLngMatches[1]) || null;
|
|
locationLong = parseFloat(locationLatLngMatches[2]) || null;
|
|
}
|
|
|
|
const data = {
|
|
url,
|
|
agencyObjectId: olxId,
|
|
originAgencyName: AD_AGENCY.OLX,
|
|
realEstateType: this.getAdCategoryId(category),
|
|
adType: parsedAdType,
|
|
title,
|
|
price: parsedPrice,
|
|
area: parsedArea,
|
|
gardenSize: parsedGardenSize,
|
|
shortDescription: descriptions.first().text(),
|
|
longDescription: descriptions.last().text(),
|
|
streetNumber: 0,
|
|
streetName: "",
|
|
locality: "",
|
|
municipality: "",
|
|
city: "",
|
|
region: "",
|
|
entity: "",
|
|
country: "",
|
|
locationLat,
|
|
locationLong,
|
|
adStatus: status
|
|
};
|
|
|
|
return data;
|
|
} catch (e) {
|
|
console.error("Exception caught: " + e.message, "\r\nURL:", url);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
//======= HELPER FUNCTIONS =============
|
|
|
|
getAdCategoryId(categoryText) {
|
|
switch (categoryText) {
|
|
case "Stanovi":
|
|
return AD_CATEGORY.CATEGORY_FLAT;
|
|
case "Zemljišta":
|
|
return AD_CATEGORY.CATEGORY_LAND;
|
|
case "Kuće":
|
|
return AD_CATEGORY.CATEGORY_HOUSE;
|
|
case "Poslovni prostori":
|
|
return AD_CATEGORY.CATEGORY_OFFICE;
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
getAdTypeId(adTypeText) {
|
|
switch (adTypeText) {
|
|
case "Prodaja":
|
|
return AD_TYPE.AD_TYPE_SALE;
|
|
case "Izdavanje":
|
|
return AD_TYPE.AD_TYPE_RENT;
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
parseArea(areaText) {
|
|
if (!areaText) {
|
|
return NaN;
|
|
}
|
|
const removeDotsExceptLastOneRegex = /[.](?=.*[.])/g;
|
|
const textWithOnlyOneDecimalDot = areaText
|
|
.replace(",", ".")
|
|
.replace(removeDotsExceptLastOneRegex, "");
|
|
|
|
return parseFloat(textWithOnlyOneDecimalDot);
|
|
}
|
|
|
|
parsePrice(priceText) {
|
|
if (!priceText) {
|
|
return NaN;
|
|
}
|
|
const formattedPriceText = priceText.replace(".", "").replace(",", ".");
|
|
return parseFloat(formattedPriceText);
|
|
}
|
|
|
|
async sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async saveCrawledResults(results) {
|
|
const savers = this.savers;
|
|
|
|
for (const saver of savers) {
|
|
await saver.save(results);
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = OlxCrawler;
|