Added delay between ads.
This commit is contained in:
Naida Vatric
@@ -35,5 +35,6 @@ module.exports = {
|
||||
OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
|
||||
OLX_DELAY_BETWEEN_PAGES:
|
||||
parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000,
|
||||
OLX_DELAY_BETWEEN_ADS: parseInt(process.env.OLX_DELAY_BETWEEN_ADS) || 1000,
|
||||
OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL)
|
||||
};
|
||||
|
||||
@@ -42,7 +42,10 @@ const OLX_ENUMS = {
|
||||
OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
|
||||
};
|
||||
|
||||
const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
|
||||
const {
|
||||
OLX_FORCE_CRAWL,
|
||||
OLX_DELAY_BETWEEN_ADS
|
||||
} = require("../specificConfigs/olx");
|
||||
|
||||
class OlxCrawler {
|
||||
constructor(
|
||||
@@ -52,7 +55,8 @@ class OlxCrawler {
|
||||
maxPages = 1000,
|
||||
maxResultsPerPage = 100,
|
||||
ignoredUsernames = [],
|
||||
delayBetweenPages = 1000
|
||||
delayBetweenPages = 1000,
|
||||
delayBetweenAds = OLX_DELAY_BETWEEN_ADS
|
||||
) {
|
||||
this.savers = savers;
|
||||
this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
|
||||
@@ -62,6 +66,7 @@ class OlxCrawler {
|
||||
this.maxResultsPerPage = maxResultsPerPage;
|
||||
this.ignoredUsernames = ignoredUsernames;
|
||||
this.delayBetweenPages = delayBetweenPages;
|
||||
this.delayBetweenAds = delayBetweenAds;
|
||||
}
|
||||
|
||||
async crawl() {
|
||||
@@ -193,6 +198,8 @@ class OlxCrawler {
|
||||
const asyncScraping = [];
|
||||
for (let i = 0; i < actualNoOfResults; i++) {
|
||||
asyncScraping.push(this.scrapeAd(hrefs[i]));
|
||||
//Delaying next scrape ad request to avoid ScraperAPI server error
|
||||
asyncScraping.push(this.sleep(this.delayBetweenAds));
|
||||
}
|
||||
|
||||
const scrapedData = await Promise.all(asyncScraping);
|
||||
@@ -206,492 +213,486 @@ class OlxCrawler {
|
||||
|
||||
async scrapeAd(url) {
|
||||
console.log("Scraping : ", url);
|
||||
let hasParseErrors = false;
|
||||
let numberOfParseErrors = 0;
|
||||
do {
|
||||
try {
|
||||
const adPageSource = await fetch(url);
|
||||
const body = await adPageSource.text();
|
||||
const $ = cheerio.load(body);
|
||||
let status = AD_STATUS.STATUS_NORMAL;
|
||||
// let hasParseErrors = false;
|
||||
//let numberOfParseErrors = 0;
|
||||
// do {
|
||||
try {
|
||||
const adPageSource = await fetch(url);
|
||||
const body = await adPageSource.text();
|
||||
const $ = cheerio.load(body);
|
||||
let status = AD_STATUS.STATUS_NORMAL;
|
||||
//
|
||||
console.log("Body:", body);
|
||||
//
|
||||
const propertySelectors = {
|
||||
username:
|
||||
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
|
||||
title: "#naslovartikla",
|
||||
descriptions: ".artikal_detaljniopis_tekst",
|
||||
category:
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
|
||||
};
|
||||
|
||||
const propertySelectors = {
|
||||
username:
|
||||
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
|
||||
title: "#naslovartikla",
|
||||
descriptions: ".artikal_detaljniopis_tekst",
|
||||
category:
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
|
||||
};
|
||||
const username = $(propertySelectors.username)
|
||||
.text()
|
||||
.trim();
|
||||
if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const username = $(propertySelectors.username)
|
||||
.text()
|
||||
const title = $(propertySelectors.title)
|
||||
.text()
|
||||
.trim();
|
||||
const descriptions = $(propertySelectors.descriptions);
|
||||
const category = $(propertySelectors.category)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//====== PRICE DETECTION AND EXTRACTION =====
|
||||
let price = null;
|
||||
let normalPrice = null;
|
||||
let urgentPrice = null;
|
||||
const normalPriceValue = $("#pc > p:nth-child(2)")
|
||||
.text()
|
||||
.trim();
|
||||
const urgentPriceValue = $(
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//Debug
|
||||
//console.log("Title:", title);
|
||||
//console.log("Url scraped:", url);
|
||||
// console.log("Normal price value:", normalPriceValue);
|
||||
// console.log("Urgent price value:", urgentPriceValue);
|
||||
//
|
||||
if (normalPriceValue && normalPriceValue.length > 0) {
|
||||
normalPrice = normalPriceValue
|
||||
.replace(/\r\n|\n|\r/gm, "")
|
||||
.replace("KM", "")
|
||||
.trim();
|
||||
if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
|
||||
return null;
|
||||
if (
|
||||
$("#pc > p.n")
|
||||
.text()
|
||||
.indexOf("Hitna") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
status = AD_STATUS.STATUS_NORMAL;
|
||||
}
|
||||
|
||||
const title = $(propertySelectors.title)
|
||||
.text()
|
||||
.trim();
|
||||
const descriptions = $(propertySelectors.descriptions);
|
||||
const category = $(propertySelectors.category)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//====== PRICE DETECTION AND EXTRACTION =====
|
||||
let price = null;
|
||||
let normalPrice = null;
|
||||
let urgentPrice = null;
|
||||
const normalPriceValue = $("#pc > p:nth-child(2)")
|
||||
.text()
|
||||
.trim();
|
||||
const urgentPriceValue = $(
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//Debug
|
||||
//console.log("Title:", title);
|
||||
//console.log("Url scraped:", url);
|
||||
// console.log("Normal price value:", normalPriceValue);
|
||||
// console.log("Urgent price value:", urgentPriceValue);
|
||||
//
|
||||
if (normalPriceValue && normalPriceValue.length > 0) {
|
||||
normalPrice = normalPriceValue
|
||||
.replace(/\r\n|\n|\r/gm, "")
|
||||
.replace("KM", "")
|
||||
.trim();
|
||||
if (
|
||||
$("#pc > p.n")
|
||||
.text()
|
||||
.indexOf("Hitna") !== -1
|
||||
) {
|
||||
} else {
|
||||
throw { message: "Can't find normal price" };
|
||||
}
|
||||
if (urgentPriceValue && urgentPriceValue.length > 0) {
|
||||
const priceValues = urgentPriceValue.replace("Cijena", "").split("KM");
|
||||
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
|
||||
if (priceValues.length > 0) {
|
||||
if (priceValues[0].trim().indexOf("Hitno") != -1) {
|
||||
urgentPrice = priceValues[0].replace("Hitno", "").trim();
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
status = AD_STATUS.STATUS_NORMAL;
|
||||
urgentPrice = priceValues[0].trim();
|
||||
}
|
||||
} else {
|
||||
throw { message: "Can't find normal price" };
|
||||
}
|
||||
if (urgentPriceValue && urgentPriceValue.length > 0) {
|
||||
const priceValues = urgentPriceValue
|
||||
.replace("Cijena", "")
|
||||
.split("KM");
|
||||
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
|
||||
if (priceValues.length > 0) {
|
||||
if (priceValues[0].trim().indexOf("Hitno") != -1) {
|
||||
urgentPrice = priceValues[0].replace("Hitno", "").trim();
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
urgentPrice = priceValues[0].trim();
|
||||
}
|
||||
} else {
|
||||
throw { message: "Can't find urgent price" };
|
||||
}
|
||||
throw { message: "Can't find urgent price" };
|
||||
}
|
||||
}
|
||||
|
||||
price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice;
|
||||
price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice;
|
||||
|
||||
//====== OTHER AD INFORMATION ===============
|
||||
let adType = null;
|
||||
let olxId = null;
|
||||
let numberOfViewsAgency = null;
|
||||
//====== OTHER AD INFORMATION ===============
|
||||
let adType = null;
|
||||
let olxId = null;
|
||||
let numberOfViewsAgency = null;
|
||||
|
||||
let otherInformationDivId;
|
||||
//We need to locate DIV ID where other information are stored
|
||||
for (let possibleId = 1; possibleId <= 30; possibleId++) {
|
||||
const adTypeFieldTitle = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (adTypeFieldTitle === "Vrsta oglasa") {
|
||||
otherInformationDivId = possibleId;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!otherInformationDivId) {
|
||||
throw { message: "Other information DIV could not be found" };
|
||||
}
|
||||
|
||||
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
|
||||
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
|
||||
const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
|
||||
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
|
||||
|
||||
const publishedDate = $(publishedDateValueSelector)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const publishedDateMoment = moment.tz(
|
||||
publishedDate,
|
||||
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!publishedDateMoment.isValid()) {
|
||||
throw { message: "Invalid published date ! Check parsing format" };
|
||||
}
|
||||
|
||||
const renewedDate = $(renewedDateFullValueSelector)
|
||||
.data("content")
|
||||
.trim();
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
renewedDate,
|
||||
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!renewedDateMoment) {
|
||||
throw {
|
||||
message:
|
||||
"Invalid renewed date ! Check how parser parsed renewed date text"
|
||||
};
|
||||
}
|
||||
|
||||
adType = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
|
||||
let otherInformationDivId;
|
||||
//We need to locate DIV ID where other information are stored
|
||||
for (let possibleId = 1; possibleId <= 30; possibleId++) {
|
||||
const adTypeFieldTitle = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const parsedCategory = this.getAdCategoryId(category);
|
||||
if (!parsedCategory) {
|
||||
throw { message: `Unknown ad category [${category}]` };
|
||||
if (adTypeFieldTitle === "Vrsta oglasa") {
|
||||
otherInformationDivId = possibleId;
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedAdType = this.getAdTypeId(adType);
|
||||
if (!parsedAdType) {
|
||||
throw { message: "Unknown ad type" };
|
||||
}
|
||||
|
||||
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
|
||||
.text()
|
||||
.trim();
|
||||
olxId = $(`${olxIdFieldSelector} > div.df2`)
|
||||
.text()
|
||||
.trim();
|
||||
numberOfViewsAgency = parseInt(
|
||||
$(numberOfViewsAgencyValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
);
|
||||
|
||||
if (olxIdFieldTitle !== "OLX ID") {
|
||||
throw { message: "Cannot find correct OLX ID" };
|
||||
}
|
||||
//===========================================
|
||||
|
||||
//====== DETAIL INFORMATION FIELDS ==========
|
||||
let area,
|
||||
gardenSize,
|
||||
numberOfRooms = null,
|
||||
numberOfFloors = null,
|
||||
floor = null,
|
||||
accessRoadType = null,
|
||||
heatingType = null,
|
||||
furnishingType = null,
|
||||
balcony = null,
|
||||
newBuilding = null,
|
||||
elevator = null,
|
||||
water = null,
|
||||
electricity = null,
|
||||
drainageSystem = null,
|
||||
registeredInZkBooks = null,
|
||||
recentlyAdapted = null,
|
||||
parking = null,
|
||||
garage = null,
|
||||
gas = null,
|
||||
antiTheftDoor = null,
|
||||
airCondition = null,
|
||||
phoneConnection = null,
|
||||
cableTV = null,
|
||||
internet = null,
|
||||
basementAttic = null,
|
||||
storeRoom = null,
|
||||
videoSurveillance = null,
|
||||
alarm = null,
|
||||
suitableForStudents = null,
|
||||
includingBills = null,
|
||||
animalsAllowed = null,
|
||||
pool = null,
|
||||
urbanPlanPermit = null,
|
||||
buildingPermit = null,
|
||||
utilityConnection = null,
|
||||
distanceToRiver = null;
|
||||
|
||||
let fieldIndex = 1;
|
||||
do {
|
||||
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
|
||||
const fieldTitleSelector = `${fieldSelector} > div.df1`;
|
||||
const fieldValueSelector = `${fieldSelector} > div.df2`;
|
||||
|
||||
const fieldTitle = $(fieldTitleSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const fieldValue = $(fieldValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
switch (fieldTitle) {
|
||||
case "kvadrata":
|
||||
area = fieldValue;
|
||||
break;
|
||||
case "okućnica (kvadratura)":
|
||||
gardenSize = fieldValue;
|
||||
break;
|
||||
case "broj soba":
|
||||
numberOfRooms = this.parseNumberOfRooms(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "broj prostorija":
|
||||
numberOfRooms = this.parseNumberOfRooms(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "broj spratova":
|
||||
numberOfFloors = this.parseNumberOfFloors(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "sprat":
|
||||
floor = this.parseFloorNumber(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "vrsta grijanja":
|
||||
heatingType = this.getHeatingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten?":
|
||||
furnishingType = this.getFurnishingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "namještena":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "voda":
|
||||
water = true;
|
||||
break;
|
||||
case "struja":
|
||||
electricity = true;
|
||||
break;
|
||||
case "kanalizacija":
|
||||
drainageSystem = fieldValue !== "nema";
|
||||
break;
|
||||
case "godina izgradnje":
|
||||
newBuilding = newBuilding || fieldValue === "novogradnja";
|
||||
break;
|
||||
case "kućni ljubimci":
|
||||
animalsAllowed = fieldValue === "da";
|
||||
break;
|
||||
case "uknjiženo / zk":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "uknjiženo (zk)":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "novogradnja":
|
||||
newBuilding = true;
|
||||
break;
|
||||
case "nedavno adaptiran":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "nedavno adaptirana":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "balkon":
|
||||
balcony = true;
|
||||
break;
|
||||
case "lift":
|
||||
elevator = true;
|
||||
break;
|
||||
case "parking":
|
||||
parking = true;
|
||||
break;
|
||||
case "garaža":
|
||||
garage = true;
|
||||
break;
|
||||
case "plin":
|
||||
gas = true;
|
||||
break;
|
||||
case "blindirana vrata":
|
||||
antiTheftDoor = true;
|
||||
break;
|
||||
case "klima":
|
||||
airCondition = true;
|
||||
break;
|
||||
case "telefonski priključak":
|
||||
phoneConnection = true;
|
||||
break;
|
||||
case "kablovska tv":
|
||||
cableTV = true;
|
||||
break;
|
||||
case "internet":
|
||||
internet = true;
|
||||
break;
|
||||
case "podrum/tavan":
|
||||
basementAttic = true;
|
||||
break;
|
||||
case "ostava/špajz":
|
||||
storeRoom = true;
|
||||
break;
|
||||
case "video nadzor":
|
||||
videoSurveillance = true;
|
||||
break;
|
||||
case "alarm":
|
||||
alarm = true;
|
||||
break;
|
||||
case "za studente":
|
||||
suitableForStudents = true;
|
||||
break;
|
||||
case "uključen trošak režija":
|
||||
includingBills = true;
|
||||
break;
|
||||
case "građevinska dozvola":
|
||||
buildingPermit = true;
|
||||
break;
|
||||
case "komunalni priključak":
|
||||
utilityConnection = true;
|
||||
break;
|
||||
case "urbanistička dozvola":
|
||||
urbanPlanPermit = true;
|
||||
break;
|
||||
case "udaljenost od rijeke (m)":
|
||||
distanceToRiver = parseInt(fieldValue) || null;
|
||||
break;
|
||||
case "prilaz":
|
||||
accessRoadType = this.getAccessRoadTypeId(fieldValue);
|
||||
break;
|
||||
case "bazen":
|
||||
pool = true;
|
||||
break;
|
||||
case "iznajmljeno":
|
||||
status = AD_STATUS.STATUS_RENTED;
|
||||
break;
|
||||
default:
|
||||
// console.log(fieldTitle, " = ", fieldValue);
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS ||
|
||||
fieldTitle === ""
|
||||
) {
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
//===========================================
|
||||
|
||||
//=========================================
|
||||
const parsedArea = this.parseArea(area) || null;
|
||||
const parsedGardenSize = this.parseArea(gardenSize) || null;
|
||||
const parsedPrice = this.parsePrice(price) || null;
|
||||
|
||||
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
||||
const locationLatLngMatches = latLngRegex.exec(body);
|
||||
|
||||
let locationLat = null;
|
||||
let locationLong = null;
|
||||
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
|
||||
locationLat = parseFloat(locationLatLngMatches[1]) || null;
|
||||
locationLong = parseFloat(locationLatLngMatches[2]) || null;
|
||||
}
|
||||
|
||||
if (
|
||||
title.indexOf("[PRODANO]") !== -1 ||
|
||||
title.indexOf("[ZAVRŠENO]") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_SOLD;
|
||||
}
|
||||
|
||||
const data = {
|
||||
url,
|
||||
agencyObjectId: olxId,
|
||||
originAgencyName: AD_AGENCY.OLX,
|
||||
realEstateType: parsedCategory,
|
||||
adType: parsedAdType,
|
||||
title,
|
||||
price: parsedPrice,
|
||||
area: parsedArea,
|
||||
gardenSize: parsedGardenSize,
|
||||
shortDescription: descriptions
|
||||
.first()
|
||||
.text()
|
||||
.trim(),
|
||||
longDescription: descriptions
|
||||
.last()
|
||||
.text()
|
||||
.trim(),
|
||||
streetNumber: 0,
|
||||
streetName: "",
|
||||
locality: "",
|
||||
municipality: "",
|
||||
city: "",
|
||||
region: "",
|
||||
entity: "",
|
||||
country: "",
|
||||
locationLat,
|
||||
locationLong,
|
||||
adStatus: status,
|
||||
publishedDate: publishedDateMoment.toISOString(),
|
||||
renewedDate: renewedDateMoment.toISOString(),
|
||||
numberOfRooms,
|
||||
numberOfFloors,
|
||||
floor,
|
||||
accessRoadType,
|
||||
heatingType,
|
||||
furnishingType,
|
||||
balcony,
|
||||
newBuilding,
|
||||
elevator,
|
||||
water,
|
||||
electricity,
|
||||
drainageSystem,
|
||||
registeredInZkBooks,
|
||||
recentlyAdapted,
|
||||
parking,
|
||||
garage,
|
||||
gas,
|
||||
antiTheftDoor,
|
||||
airCondition,
|
||||
phoneConnection,
|
||||
cableTV,
|
||||
internet,
|
||||
basementAttic,
|
||||
storeRoom,
|
||||
videoSurveillance,
|
||||
alarm,
|
||||
suitableForStudents,
|
||||
includingBills,
|
||||
animalsAllowed,
|
||||
pool,
|
||||
urbanPlanPermit,
|
||||
buildingPermit,
|
||||
utilityConnection,
|
||||
distanceToRiver,
|
||||
numberOfViewsAgency
|
||||
};
|
||||
//
|
||||
//console.log("Scraped data:", data);
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
hasParseErrors = true;
|
||||
numberOfParseErrors++;
|
||||
console.error("Exception caught: " + e.message, "\r\nURL:", url);
|
||||
}
|
||||
} while (hasParseErrors && numberOfParseErrors <= 1);
|
||||
|
||||
if (!otherInformationDivId) {
|
||||
throw { message: "Other information DIV could not be found" };
|
||||
}
|
||||
|
||||
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
|
||||
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
|
||||
const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
|
||||
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
|
||||
|
||||
const publishedDate = $(publishedDateValueSelector)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const publishedDateMoment = moment.tz(
|
||||
publishedDate,
|
||||
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!publishedDateMoment.isValid()) {
|
||||
throw { message: "Invalid published date ! Check parsing format" };
|
||||
}
|
||||
|
||||
const renewedDate = $(renewedDateFullValueSelector)
|
||||
.data("content")
|
||||
.trim();
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
renewedDate,
|
||||
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!renewedDateMoment) {
|
||||
throw {
|
||||
message:
|
||||
"Invalid renewed date ! Check how parser parsed renewed date text"
|
||||
};
|
||||
}
|
||||
|
||||
adType = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const parsedCategory = this.getAdCategoryId(category);
|
||||
if (!parsedCategory) {
|
||||
throw { message: `Unknown ad category [${category}]` };
|
||||
}
|
||||
|
||||
const parsedAdType = this.getAdTypeId(adType);
|
||||
if (!parsedAdType) {
|
||||
throw { message: "Unknown ad type" };
|
||||
}
|
||||
|
||||
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
|
||||
.text()
|
||||
.trim();
|
||||
olxId = $(`${olxIdFieldSelector} > div.df2`)
|
||||
.text()
|
||||
.trim();
|
||||
numberOfViewsAgency = parseInt(
|
||||
$(numberOfViewsAgencyValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
);
|
||||
|
||||
if (olxIdFieldTitle !== "OLX ID") {
|
||||
throw { message: "Cannot find correct OLX ID" };
|
||||
}
|
||||
//===========================================
|
||||
|
||||
//====== DETAIL INFORMATION FIELDS ==========
|
||||
let area,
|
||||
gardenSize,
|
||||
numberOfRooms = null,
|
||||
numberOfFloors = null,
|
||||
floor = null,
|
||||
accessRoadType = null,
|
||||
heatingType = null,
|
||||
furnishingType = null,
|
||||
balcony = null,
|
||||
newBuilding = null,
|
||||
elevator = null,
|
||||
water = null,
|
||||
electricity = null,
|
||||
drainageSystem = null,
|
||||
registeredInZkBooks = null,
|
||||
recentlyAdapted = null,
|
||||
parking = null,
|
||||
garage = null,
|
||||
gas = null,
|
||||
antiTheftDoor = null,
|
||||
airCondition = null,
|
||||
phoneConnection = null,
|
||||
cableTV = null,
|
||||
internet = null,
|
||||
basementAttic = null,
|
||||
storeRoom = null,
|
||||
videoSurveillance = null,
|
||||
alarm = null,
|
||||
suitableForStudents = null,
|
||||
includingBills = null,
|
||||
animalsAllowed = null,
|
||||
pool = null,
|
||||
urbanPlanPermit = null,
|
||||
buildingPermit = null,
|
||||
utilityConnection = null,
|
||||
distanceToRiver = null;
|
||||
|
||||
let fieldIndex = 1;
|
||||
do {
|
||||
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
|
||||
const fieldTitleSelector = `${fieldSelector} > div.df1`;
|
||||
const fieldValueSelector = `${fieldSelector} > div.df2`;
|
||||
|
||||
const fieldTitle = $(fieldTitleSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const fieldValue = $(fieldValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
switch (fieldTitle) {
|
||||
case "kvadrata":
|
||||
area = fieldValue;
|
||||
break;
|
||||
case "okućnica (kvadratura)":
|
||||
gardenSize = fieldValue;
|
||||
break;
|
||||
case "broj soba":
|
||||
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "broj prostorija":
|
||||
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "broj spratova":
|
||||
numberOfFloors = this.parseNumberOfFloors(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "sprat":
|
||||
floor = this.parseFloorNumber(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "vrsta grijanja":
|
||||
heatingType = this.getHeatingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten?":
|
||||
furnishingType = this.getFurnishingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "namještena":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "voda":
|
||||
water = true;
|
||||
break;
|
||||
case "struja":
|
||||
electricity = true;
|
||||
break;
|
||||
case "kanalizacija":
|
||||
drainageSystem = fieldValue !== "nema";
|
||||
break;
|
||||
case "godina izgradnje":
|
||||
newBuilding = newBuilding || fieldValue === "novogradnja";
|
||||
break;
|
||||
case "kućni ljubimci":
|
||||
animalsAllowed = fieldValue === "da";
|
||||
break;
|
||||
case "uknjiženo / zk":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "uknjiženo (zk)":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "novogradnja":
|
||||
newBuilding = true;
|
||||
break;
|
||||
case "nedavno adaptiran":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "nedavno adaptirana":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "balkon":
|
||||
balcony = true;
|
||||
break;
|
||||
case "lift":
|
||||
elevator = true;
|
||||
break;
|
||||
case "parking":
|
||||
parking = true;
|
||||
break;
|
||||
case "garaža":
|
||||
garage = true;
|
||||
break;
|
||||
case "plin":
|
||||
gas = true;
|
||||
break;
|
||||
case "blindirana vrata":
|
||||
antiTheftDoor = true;
|
||||
break;
|
||||
case "klima":
|
||||
airCondition = true;
|
||||
break;
|
||||
case "telefonski priključak":
|
||||
phoneConnection = true;
|
||||
break;
|
||||
case "kablovska tv":
|
||||
cableTV = true;
|
||||
break;
|
||||
case "internet":
|
||||
internet = true;
|
||||
break;
|
||||
case "podrum/tavan":
|
||||
basementAttic = true;
|
||||
break;
|
||||
case "ostava/špajz":
|
||||
storeRoom = true;
|
||||
break;
|
||||
case "video nadzor":
|
||||
videoSurveillance = true;
|
||||
break;
|
||||
case "alarm":
|
||||
alarm = true;
|
||||
break;
|
||||
case "za studente":
|
||||
suitableForStudents = true;
|
||||
break;
|
||||
case "uključen trošak režija":
|
||||
includingBills = true;
|
||||
break;
|
||||
case "građevinska dozvola":
|
||||
buildingPermit = true;
|
||||
break;
|
||||
case "komunalni priključak":
|
||||
utilityConnection = true;
|
||||
break;
|
||||
case "urbanistička dozvola":
|
||||
urbanPlanPermit = true;
|
||||
break;
|
||||
case "udaljenost od rijeke (m)":
|
||||
distanceToRiver = parseInt(fieldValue) || null;
|
||||
break;
|
||||
case "prilaz":
|
||||
accessRoadType = this.getAccessRoadTypeId(fieldValue);
|
||||
break;
|
||||
case "bazen":
|
||||
pool = true;
|
||||
break;
|
||||
case "iznajmljeno":
|
||||
status = AD_STATUS.STATUS_RENTED;
|
||||
break;
|
||||
default:
|
||||
// console.log(fieldTitle, " = ", fieldValue);
|
||||
break;
|
||||
}
|
||||
|
||||
if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
//===========================================
|
||||
|
||||
//=========================================
|
||||
const parsedArea = this.parseArea(area) || null;
|
||||
const parsedGardenSize = this.parseArea(gardenSize) || null;
|
||||
const parsedPrice = this.parsePrice(price) || null;
|
||||
|
||||
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
||||
const locationLatLngMatches = latLngRegex.exec(body);
|
||||
|
||||
let locationLat = null;
|
||||
let locationLong = null;
|
||||
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
|
||||
locationLat = parseFloat(locationLatLngMatches[1]) || null;
|
||||
locationLong = parseFloat(locationLatLngMatches[2]) || null;
|
||||
}
|
||||
|
||||
if (
|
||||
title.indexOf("[PRODANO]") !== -1 ||
|
||||
title.indexOf("[ZAVRŠENO]") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_SOLD;
|
||||
}
|
||||
|
||||
const data = {
|
||||
url,
|
||||
agencyObjectId: olxId,
|
||||
originAgencyName: AD_AGENCY.OLX,
|
||||
realEstateType: parsedCategory,
|
||||
adType: parsedAdType,
|
||||
title,
|
||||
price: parsedPrice,
|
||||
area: parsedArea,
|
||||
gardenSize: parsedGardenSize,
|
||||
shortDescription: descriptions
|
||||
.first()
|
||||
.text()
|
||||
.trim(),
|
||||
longDescription: descriptions
|
||||
.last()
|
||||
.text()
|
||||
.trim(),
|
||||
streetNumber: 0,
|
||||
streetName: "",
|
||||
locality: "",
|
||||
municipality: "",
|
||||
city: "",
|
||||
region: "",
|
||||
entity: "",
|
||||
country: "",
|
||||
locationLat,
|
||||
locationLong,
|
||||
adStatus: status,
|
||||
publishedDate: publishedDateMoment.toISOString(),
|
||||
renewedDate: renewedDateMoment.toISOString(),
|
||||
numberOfRooms,
|
||||
numberOfFloors,
|
||||
floor,
|
||||
accessRoadType,
|
||||
heatingType,
|
||||
furnishingType,
|
||||
balcony,
|
||||
newBuilding,
|
||||
elevator,
|
||||
water,
|
||||
electricity,
|
||||
drainageSystem,
|
||||
registeredInZkBooks,
|
||||
recentlyAdapted,
|
||||
parking,
|
||||
garage,
|
||||
gas,
|
||||
antiTheftDoor,
|
||||
airCondition,
|
||||
phoneConnection,
|
||||
cableTV,
|
||||
internet,
|
||||
basementAttic,
|
||||
storeRoom,
|
||||
videoSurveillance,
|
||||
alarm,
|
||||
suitableForStudents,
|
||||
includingBills,
|
||||
animalsAllowed,
|
||||
pool,
|
||||
urbanPlanPermit,
|
||||
buildingPermit,
|
||||
utilityConnection,
|
||||
distanceToRiver,
|
||||
numberOfViewsAgency
|
||||
};
|
||||
//
|
||||
//console.log("Scraped data:", data);
|
||||
|
||||
//Delay between real estate ads to avoid error from Scraper API
|
||||
// await this.sleep(this.delayBetweenAds);
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
// hasParseErrors = true;
|
||||
// numberOfParseErrors++;
|
||||
console.error("Exception caught: " + e.message, "\r\nURL:", url);
|
||||
}
|
||||
// } while (hasParseErrors && numberOfParseErrors <= 1);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -909,6 +910,7 @@ class OlxCrawler {
|
||||
}
|
||||
|
||||
async sleep(ms) {
|
||||
// console.log("Sleep for:", ms);
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,9 @@ OLX_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check commo
|
||||
OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
|
||||
OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
|
||||
OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
|
||||
OLX_DELAY_BETWEEN_ADS = time in miliseconds to wait before scraping next add to awoid server errors with Scraper API
|
||||
OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
|
||||
|
||||
#==RENTAL==
|
||||
RENTAL_MAX_PAGES=Restrict crawler to this number of pages
|
||||
RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
|
||||
|
||||
Reference in New Issue
Block a user