Olx sraper scrape twice.
This commit is contained in:
Naida Vatric
@@ -206,475 +206,492 @@ class OlxCrawler {
|
||||
|
||||
async scrapeAd(url) {
|
||||
console.log("Scraping : ", url);
|
||||
try {
|
||||
const adPageSource = await fetch(url);
|
||||
const body = await adPageSource.text();
|
||||
const $ = cheerio.load(body);
|
||||
let status = AD_STATUS.STATUS_NORMAL;
|
||||
let hasParseErrors = false;
|
||||
let numberOfParseErrors = 0;
|
||||
do {
|
||||
try {
|
||||
const adPageSource = await fetch(url);
|
||||
const body = await adPageSource.text();
|
||||
const $ = cheerio.load(body);
|
||||
let status = AD_STATUS.STATUS_NORMAL;
|
||||
|
||||
const propertySelectors = {
|
||||
username:
|
||||
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
|
||||
title: "#naslovartikla",
|
||||
descriptions: ".artikal_detaljniopis_tekst",
|
||||
category:
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
|
||||
};
|
||||
const propertySelectors = {
|
||||
username:
|
||||
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
|
||||
title: "#naslovartikla",
|
||||
descriptions: ".artikal_detaljniopis_tekst",
|
||||
category:
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
|
||||
};
|
||||
|
||||
const username = $(propertySelectors.username)
|
||||
.text()
|
||||
.trim();
|
||||
if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const title = $(propertySelectors.title)
|
||||
.text()
|
||||
.trim();
|
||||
const descriptions = $(propertySelectors.descriptions);
|
||||
const category = $(propertySelectors.category)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//====== PRICE DETECTION AND EXTRACTION =====
|
||||
let price = null;
|
||||
let normalPrice = null;
|
||||
let urgentPrice = null;
|
||||
const normalPriceValue = $("#pc > p:nth-child(2)")
|
||||
.text()
|
||||
.trim();
|
||||
const urgentPriceValue = $(
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//Debug
|
||||
//console.log("Title:", title);
|
||||
//console.log("Url scraped:", url);
|
||||
// console.log("Normal price value:", normalPriceValue);
|
||||
// console.log("Urgent price value:", urgentPriceValue);
|
||||
//
|
||||
if (normalPriceValue && normalPriceValue.length > 0) {
|
||||
normalPrice = normalPriceValue
|
||||
.replace(/\r\n|\n|\r/gm, "")
|
||||
.replace("KM", "")
|
||||
const username = $(propertySelectors.username)
|
||||
.text()
|
||||
.trim();
|
||||
if (
|
||||
$("#pc > p.n")
|
||||
.text()
|
||||
.indexOf("Hitna") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
status = AD_STATUS.STATUS_NORMAL;
|
||||
if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
throw { message: "Can't find normal price" };
|
||||
}
|
||||
if (urgentPriceValue && urgentPriceValue.length > 0) {
|
||||
const priceValues = urgentPriceValue.replace("Cijena", "").split("KM");
|
||||
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
|
||||
if (priceValues.length > 0) {
|
||||
if (priceValues[0].trim().indexOf("Hitno") != -1) {
|
||||
urgentPrice = priceValues[0].replace("Hitno", "").trim();
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
urgentPrice = priceValues[0].trim();
|
||||
}
|
||||
} else {
|
||||
throw { message: "Can't find urgent price" };
|
||||
}
|
||||
}
|
||||
|
||||
price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice;
|
||||
const title = $(propertySelectors.title)
|
||||
.text()
|
||||
.trim();
|
||||
const descriptions = $(propertySelectors.descriptions);
|
||||
const category = $(propertySelectors.category)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
//====== OTHER AD INFORMATION ===============
|
||||
let adType = null;
|
||||
let olxId = null;
|
||||
let numberOfViewsAgency = null;
|
||||
|
||||
let otherInformationDivId;
|
||||
//We need to locate DIV ID where other information are stored
|
||||
for (let possibleId = 1; possibleId <= 30; possibleId++) {
|
||||
const adTypeFieldTitle = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
|
||||
//====== PRICE DETECTION AND EXTRACTION =====
|
||||
let price = null;
|
||||
let normalPrice = null;
|
||||
let urgentPrice = null;
|
||||
const normalPriceValue = $("#pc > p:nth-child(2)")
|
||||
.text()
|
||||
.trim();
|
||||
const urgentPriceValue = $(
|
||||
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (adTypeFieldTitle === "Vrsta oglasa") {
|
||||
otherInformationDivId = possibleId;
|
||||
break;
|
||||
//Debug
|
||||
//console.log("Title:", title);
|
||||
//console.log("Url scraped:", url);
|
||||
// console.log("Normal price value:", normalPriceValue);
|
||||
// console.log("Urgent price value:", urgentPriceValue);
|
||||
//
|
||||
if (normalPriceValue && normalPriceValue.length > 0) {
|
||||
normalPrice = normalPriceValue
|
||||
.replace(/\r\n|\n|\r/gm, "")
|
||||
.replace("KM", "")
|
||||
.trim();
|
||||
if (
|
||||
$("#pc > p.n")
|
||||
.text()
|
||||
.indexOf("Hitna") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
status = AD_STATUS.STATUS_NORMAL;
|
||||
}
|
||||
} else {
|
||||
throw { message: "Can't find normal price" };
|
||||
}
|
||||
if (urgentPriceValue && urgentPriceValue.length > 0) {
|
||||
const priceValues = urgentPriceValue
|
||||
.replace("Cijena", "")
|
||||
.split("KM");
|
||||
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
|
||||
if (priceValues.length > 0) {
|
||||
if (priceValues[0].trim().indexOf("Hitno") != -1) {
|
||||
urgentPrice = priceValues[0].replace("Hitno", "").trim();
|
||||
status = AD_STATUS.STATUS_URGENT;
|
||||
} else {
|
||||
urgentPrice = priceValues[0].trim();
|
||||
}
|
||||
} else {
|
||||
throw { message: "Can't find urgent price" };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!otherInformationDivId) {
|
||||
throw { message: "Other information DIV could not be found" };
|
||||
}
|
||||
price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice;
|
||||
|
||||
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
|
||||
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
|
||||
const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
|
||||
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
|
||||
//====== OTHER AD INFORMATION ===============
|
||||
let adType = null;
|
||||
let olxId = null;
|
||||
let numberOfViewsAgency = null;
|
||||
|
||||
const publishedDate = $(publishedDateValueSelector)
|
||||
.text()
|
||||
.trim();
|
||||
let otherInformationDivId;
|
||||
//We need to locate DIV ID where other information are stored
|
||||
for (let possibleId = 1; possibleId <= 30; possibleId++) {
|
||||
const adTypeFieldTitle = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const publishedDateMoment = moment.tz(
|
||||
publishedDate,
|
||||
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
if (adTypeFieldTitle === "Vrsta oglasa") {
|
||||
otherInformationDivId = possibleId;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!publishedDateMoment.isValid()) {
|
||||
throw { message: "Invalid published date ! Check parsing format" };
|
||||
}
|
||||
if (!otherInformationDivId) {
|
||||
throw { message: "Other information DIV could not be found" };
|
||||
}
|
||||
|
||||
const renewedDate = $(renewedDateFullValueSelector)
|
||||
.data("content")
|
||||
.trim();
|
||||
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
|
||||
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
|
||||
const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
|
||||
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
renewedDate,
|
||||
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
const publishedDate = $(publishedDateValueSelector)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (!renewedDateMoment) {
|
||||
throw {
|
||||
message:
|
||||
"Invalid renewed date ! Check how parser parsed renewed date text"
|
||||
const publishedDateMoment = moment.tz(
|
||||
publishedDate,
|
||||
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!publishedDateMoment.isValid()) {
|
||||
throw { message: "Invalid published date ! Check parsing format" };
|
||||
}
|
||||
|
||||
const renewedDate = $(renewedDateFullValueSelector)
|
||||
.data("content")
|
||||
.trim();
|
||||
|
||||
const renewedDateMoment = moment.tz(
|
||||
renewedDate,
|
||||
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
|
||||
DEFAULT_TIMEZONE
|
||||
);
|
||||
|
||||
if (!renewedDateMoment) {
|
||||
throw {
|
||||
message:
|
||||
"Invalid renewed date ! Check how parser parsed renewed date text"
|
||||
};
|
||||
}
|
||||
|
||||
adType = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const parsedCategory = this.getAdCategoryId(category);
|
||||
if (!parsedCategory) {
|
||||
throw { message: `Unknown ad category [${category}]` };
|
||||
}
|
||||
|
||||
const parsedAdType = this.getAdTypeId(adType);
|
||||
if (!parsedAdType) {
|
||||
throw { message: "Unknown ad type" };
|
||||
}
|
||||
|
||||
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
|
||||
.text()
|
||||
.trim();
|
||||
olxId = $(`${olxIdFieldSelector} > div.df2`)
|
||||
.text()
|
||||
.trim();
|
||||
numberOfViewsAgency = parseInt(
|
||||
$(numberOfViewsAgencyValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
);
|
||||
|
||||
if (olxIdFieldTitle !== "OLX ID") {
|
||||
throw { message: "Cannot find correct OLX ID" };
|
||||
}
|
||||
//===========================================
|
||||
|
||||
//====== DETAIL INFORMATION FIELDS ==========
|
||||
let area,
|
||||
gardenSize,
|
||||
numberOfRooms = null,
|
||||
numberOfFloors = null,
|
||||
floor = null,
|
||||
accessRoadType = null,
|
||||
heatingType = null,
|
||||
furnishingType = null,
|
||||
balcony = null,
|
||||
newBuilding = null,
|
||||
elevator = null,
|
||||
water = null,
|
||||
electricity = null,
|
||||
drainageSystem = null,
|
||||
registeredInZkBooks = null,
|
||||
recentlyAdapted = null,
|
||||
parking = null,
|
||||
garage = null,
|
||||
gas = null,
|
||||
antiTheftDoor = null,
|
||||
airCondition = null,
|
||||
phoneConnection = null,
|
||||
cableTV = null,
|
||||
internet = null,
|
||||
basementAttic = null,
|
||||
storeRoom = null,
|
||||
videoSurveillance = null,
|
||||
alarm = null,
|
||||
suitableForStudents = null,
|
||||
includingBills = null,
|
||||
animalsAllowed = null,
|
||||
pool = null,
|
||||
urbanPlanPermit = null,
|
||||
buildingPermit = null,
|
||||
utilityConnection = null,
|
||||
distanceToRiver = null;
|
||||
|
||||
let fieldIndex = 1;
|
||||
do {
|
||||
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
|
||||
const fieldTitleSelector = `${fieldSelector} > div.df1`;
|
||||
const fieldValueSelector = `${fieldSelector} > div.df2`;
|
||||
|
||||
const fieldTitle = $(fieldTitleSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const fieldValue = $(fieldValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
switch (fieldTitle) {
|
||||
case "kvadrata":
|
||||
area = fieldValue;
|
||||
break;
|
||||
case "okućnica (kvadratura)":
|
||||
gardenSize = fieldValue;
|
||||
break;
|
||||
case "broj soba":
|
||||
numberOfRooms = this.parseNumberOfRooms(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "broj prostorija":
|
||||
numberOfRooms = this.parseNumberOfRooms(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "broj spratova":
|
||||
numberOfFloors = this.parseNumberOfFloors(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "sprat":
|
||||
floor = this.parseFloorNumber(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "vrsta grijanja":
|
||||
heatingType = this.getHeatingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten?":
|
||||
furnishingType = this.getFurnishingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "namještena":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "voda":
|
||||
water = true;
|
||||
break;
|
||||
case "struja":
|
||||
electricity = true;
|
||||
break;
|
||||
case "kanalizacija":
|
||||
drainageSystem = fieldValue !== "nema";
|
||||
break;
|
||||
case "godina izgradnje":
|
||||
newBuilding = newBuilding || fieldValue === "novogradnja";
|
||||
break;
|
||||
case "kućni ljubimci":
|
||||
animalsAllowed = fieldValue === "da";
|
||||
break;
|
||||
case "uknjiženo / zk":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "uknjiženo (zk)":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "novogradnja":
|
||||
newBuilding = true;
|
||||
break;
|
||||
case "nedavno adaptiran":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "nedavno adaptirana":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "balkon":
|
||||
balcony = true;
|
||||
break;
|
||||
case "lift":
|
||||
elevator = true;
|
||||
break;
|
||||
case "parking":
|
||||
parking = true;
|
||||
break;
|
||||
case "garaža":
|
||||
garage = true;
|
||||
break;
|
||||
case "plin":
|
||||
gas = true;
|
||||
break;
|
||||
case "blindirana vrata":
|
||||
antiTheftDoor = true;
|
||||
break;
|
||||
case "klima":
|
||||
airCondition = true;
|
||||
break;
|
||||
case "telefonski priključak":
|
||||
phoneConnection = true;
|
||||
break;
|
||||
case "kablovska tv":
|
||||
cableTV = true;
|
||||
break;
|
||||
case "internet":
|
||||
internet = true;
|
||||
break;
|
||||
case "podrum/tavan":
|
||||
basementAttic = true;
|
||||
break;
|
||||
case "ostava/špajz":
|
||||
storeRoom = true;
|
||||
break;
|
||||
case "video nadzor":
|
||||
videoSurveillance = true;
|
||||
break;
|
||||
case "alarm":
|
||||
alarm = true;
|
||||
break;
|
||||
case "za studente":
|
||||
suitableForStudents = true;
|
||||
break;
|
||||
case "uključen trošak režija":
|
||||
includingBills = true;
|
||||
break;
|
||||
case "građevinska dozvola":
|
||||
buildingPermit = true;
|
||||
break;
|
||||
case "komunalni priključak":
|
||||
utilityConnection = true;
|
||||
break;
|
||||
case "urbanistička dozvola":
|
||||
urbanPlanPermit = true;
|
||||
break;
|
||||
case "udaljenost od rijeke (m)":
|
||||
distanceToRiver = parseInt(fieldValue) || null;
|
||||
break;
|
||||
case "prilaz":
|
||||
accessRoadType = this.getAccessRoadTypeId(fieldValue);
|
||||
break;
|
||||
case "bazen":
|
||||
pool = true;
|
||||
break;
|
||||
case "iznajmljeno":
|
||||
status = AD_STATUS.STATUS_RENTED;
|
||||
break;
|
||||
default:
|
||||
// console.log(fieldTitle, " = ", fieldValue);
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS ||
|
||||
fieldTitle === ""
|
||||
) {
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
//===========================================
|
||||
|
||||
//=========================================
|
||||
const parsedArea = this.parseArea(area) || null;
|
||||
const parsedGardenSize = this.parseArea(gardenSize) || null;
|
||||
const parsedPrice = this.parsePrice(price) || null;
|
||||
|
||||
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
||||
const locationLatLngMatches = latLngRegex.exec(body);
|
||||
|
||||
let locationLat = null;
|
||||
let locationLong = null;
|
||||
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
|
||||
locationLat = parseFloat(locationLatLngMatches[1]) || null;
|
||||
locationLong = parseFloat(locationLatLngMatches[2]) || null;
|
||||
}
|
||||
|
||||
if (
|
||||
title.indexOf("[PRODANO]") !== -1 ||
|
||||
title.indexOf("[ZAVRŠENO]") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_SOLD;
|
||||
}
|
||||
|
||||
const data = {
|
||||
url,
|
||||
agencyObjectId: olxId,
|
||||
originAgencyName: AD_AGENCY.OLX,
|
||||
realEstateType: parsedCategory,
|
||||
adType: parsedAdType,
|
||||
title,
|
||||
price: parsedPrice,
|
||||
area: parsedArea,
|
||||
gardenSize: parsedGardenSize,
|
||||
shortDescription: descriptions
|
||||
.first()
|
||||
.text()
|
||||
.trim(),
|
||||
longDescription: descriptions
|
||||
.last()
|
||||
.text()
|
||||
.trim(),
|
||||
streetNumber: 0,
|
||||
streetName: "",
|
||||
locality: "",
|
||||
municipality: "",
|
||||
city: "",
|
||||
region: "",
|
||||
entity: "",
|
||||
country: "",
|
||||
locationLat,
|
||||
locationLong,
|
||||
adStatus: status,
|
||||
publishedDate: publishedDateMoment.toISOString(),
|
||||
renewedDate: renewedDateMoment.toISOString(),
|
||||
numberOfRooms,
|
||||
numberOfFloors,
|
||||
floor,
|
||||
accessRoadType,
|
||||
heatingType,
|
||||
furnishingType,
|
||||
balcony,
|
||||
newBuilding,
|
||||
elevator,
|
||||
water,
|
||||
electricity,
|
||||
drainageSystem,
|
||||
registeredInZkBooks,
|
||||
recentlyAdapted,
|
||||
parking,
|
||||
garage,
|
||||
gas,
|
||||
antiTheftDoor,
|
||||
airCondition,
|
||||
phoneConnection,
|
||||
cableTV,
|
||||
internet,
|
||||
basementAttic,
|
||||
storeRoom,
|
||||
videoSurveillance,
|
||||
alarm,
|
||||
suitableForStudents,
|
||||
includingBills,
|
||||
animalsAllowed,
|
||||
pool,
|
||||
urbanPlanPermit,
|
||||
buildingPermit,
|
||||
utilityConnection,
|
||||
distanceToRiver,
|
||||
numberOfViewsAgency
|
||||
};
|
||||
//
|
||||
//console.log("Scraped data:", data);
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
hasParseErrors = true;
|
||||
numberOfParseErrors++;
|
||||
console.error("Exception caught: " + e.message, "\r\nURL:", url);
|
||||
}
|
||||
|
||||
adType = $(
|
||||
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
const parsedCategory = this.getAdCategoryId(category);
|
||||
if (!parsedCategory) {
|
||||
throw { message: `Unknown ad category [${category}]` };
|
||||
}
|
||||
|
||||
const parsedAdType = this.getAdTypeId(adType);
|
||||
if (!parsedAdType) {
|
||||
throw { message: "Unknown ad type" };
|
||||
}
|
||||
|
||||
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
|
||||
.text()
|
||||
.trim();
|
||||
olxId = $(`${olxIdFieldSelector} > div.df2`)
|
||||
.text()
|
||||
.trim();
|
||||
numberOfViewsAgency = parseInt(
|
||||
$(numberOfViewsAgencyValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
);
|
||||
|
||||
if (olxIdFieldTitle !== "OLX ID") {
|
||||
throw { message: "Cannot find correct OLX ID" };
|
||||
}
|
||||
//===========================================
|
||||
|
||||
//====== DETAIL INFORMATION FIELDS ==========
|
||||
let area,
|
||||
gardenSize,
|
||||
numberOfRooms = null,
|
||||
numberOfFloors = null,
|
||||
floor = null,
|
||||
accessRoadType = null,
|
||||
heatingType = null,
|
||||
furnishingType = null,
|
||||
balcony = null,
|
||||
newBuilding = null,
|
||||
elevator = null,
|
||||
water = null,
|
||||
electricity = null,
|
||||
drainageSystem = null,
|
||||
registeredInZkBooks = null,
|
||||
recentlyAdapted = null,
|
||||
parking = null,
|
||||
garage = null,
|
||||
gas = null,
|
||||
antiTheftDoor = null,
|
||||
airCondition = null,
|
||||
phoneConnection = null,
|
||||
cableTV = null,
|
||||
internet = null,
|
||||
basementAttic = null,
|
||||
storeRoom = null,
|
||||
videoSurveillance = null,
|
||||
alarm = null,
|
||||
suitableForStudents = null,
|
||||
includingBills = null,
|
||||
animalsAllowed = null,
|
||||
pool = null,
|
||||
urbanPlanPermit = null,
|
||||
buildingPermit = null,
|
||||
utilityConnection = null,
|
||||
distanceToRiver = null;
|
||||
|
||||
let fieldIndex = 1;
|
||||
do {
|
||||
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
|
||||
const fieldTitleSelector = `${fieldSelector} > div.df1`;
|
||||
const fieldValueSelector = `${fieldSelector} > div.df2`;
|
||||
|
||||
const fieldTitle = $(fieldTitleSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const fieldValue = $(fieldValueSelector)
|
||||
.text()
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
switch (fieldTitle) {
|
||||
case "kvadrata":
|
||||
area = fieldValue;
|
||||
break;
|
||||
case "okućnica (kvadratura)":
|
||||
gardenSize = fieldValue;
|
||||
break;
|
||||
case "broj soba":
|
||||
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "broj prostorija":
|
||||
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "broj spratova":
|
||||
numberOfFloors = this.parseNumberOfFloors(
|
||||
fieldValue,
|
||||
parsedCategory
|
||||
);
|
||||
break;
|
||||
case "sprat":
|
||||
floor = this.parseFloorNumber(fieldValue, parsedCategory);
|
||||
break;
|
||||
case "vrsta grijanja":
|
||||
heatingType = this.getHeatingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten?":
|
||||
furnishingType = this.getFurnishingTypeId(fieldValue);
|
||||
break;
|
||||
case "namješten":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "namještena":
|
||||
furnishingType = FURNISHING_TYPE.FURNISHED.id;
|
||||
break;
|
||||
case "voda":
|
||||
water = true;
|
||||
break;
|
||||
case "struja":
|
||||
electricity = true;
|
||||
break;
|
||||
case "kanalizacija":
|
||||
drainageSystem = fieldValue !== "nema";
|
||||
break;
|
||||
case "godina izgradnje":
|
||||
newBuilding = newBuilding || fieldValue === "novogradnja";
|
||||
break;
|
||||
case "kućni ljubimci":
|
||||
animalsAllowed = fieldValue === "da";
|
||||
break;
|
||||
case "uknjiženo / zk":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "uknjiženo (zk)":
|
||||
registeredInZkBooks = true;
|
||||
break;
|
||||
case "novogradnja":
|
||||
newBuilding = true;
|
||||
break;
|
||||
case "nedavno adaptiran":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "nedavno adaptirana":
|
||||
recentlyAdapted = true;
|
||||
break;
|
||||
case "balkon":
|
||||
balcony = true;
|
||||
break;
|
||||
case "lift":
|
||||
elevator = true;
|
||||
break;
|
||||
case "parking":
|
||||
parking = true;
|
||||
break;
|
||||
case "garaža":
|
||||
garage = true;
|
||||
break;
|
||||
case "plin":
|
||||
gas = true;
|
||||
break;
|
||||
case "blindirana vrata":
|
||||
antiTheftDoor = true;
|
||||
break;
|
||||
case "klima":
|
||||
airCondition = true;
|
||||
break;
|
||||
case "telefonski priključak":
|
||||
phoneConnection = true;
|
||||
break;
|
||||
case "kablovska tv":
|
||||
cableTV = true;
|
||||
break;
|
||||
case "internet":
|
||||
internet = true;
|
||||
break;
|
||||
case "podrum/tavan":
|
||||
basementAttic = true;
|
||||
break;
|
||||
case "ostava/špajz":
|
||||
storeRoom = true;
|
||||
break;
|
||||
case "video nadzor":
|
||||
videoSurveillance = true;
|
||||
break;
|
||||
case "alarm":
|
||||
alarm = true;
|
||||
break;
|
||||
case "za studente":
|
||||
suitableForStudents = true;
|
||||
break;
|
||||
case "uključen trošak režija":
|
||||
includingBills = true;
|
||||
break;
|
||||
case "građevinska dozvola":
|
||||
buildingPermit = true;
|
||||
break;
|
||||
case "komunalni priključak":
|
||||
utilityConnection = true;
|
||||
break;
|
||||
case "urbanistička dozvola":
|
||||
urbanPlanPermit = true;
|
||||
break;
|
||||
case "udaljenost od rijeke (m)":
|
||||
distanceToRiver = parseInt(fieldValue) || null;
|
||||
break;
|
||||
case "prilaz":
|
||||
accessRoadType = this.getAccessRoadTypeId(fieldValue);
|
||||
break;
|
||||
case "bazen":
|
||||
pool = true;
|
||||
break;
|
||||
case "iznajmljeno":
|
||||
status = AD_STATUS.STATUS_RENTED;
|
||||
break;
|
||||
default:
|
||||
// console.log(fieldTitle, " = ", fieldValue);
|
||||
break;
|
||||
}
|
||||
|
||||
if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
//===========================================
|
||||
|
||||
//=========================================
|
||||
const parsedArea = this.parseArea(area) || null;
|
||||
const parsedGardenSize = this.parseArea(gardenSize) || null;
|
||||
const parsedPrice = this.parsePrice(price) || null;
|
||||
|
||||
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
||||
const locationLatLngMatches = latLngRegex.exec(body);
|
||||
|
||||
let locationLat = null;
|
||||
let locationLong = null;
|
||||
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
|
||||
locationLat = parseFloat(locationLatLngMatches[1]) || null;
|
||||
locationLong = parseFloat(locationLatLngMatches[2]) || null;
|
||||
}
|
||||
|
||||
if (
|
||||
title.indexOf("[PRODANO]") !== -1 ||
|
||||
title.indexOf("[ZAVRŠENO]") !== -1
|
||||
) {
|
||||
status = AD_STATUS.STATUS_SOLD;
|
||||
}
|
||||
|
||||
const data = {
|
||||
url,
|
||||
agencyObjectId: olxId,
|
||||
originAgencyName: AD_AGENCY.OLX,
|
||||
realEstateType: parsedCategory,
|
||||
adType: parsedAdType,
|
||||
title,
|
||||
price: parsedPrice,
|
||||
area: parsedArea,
|
||||
gardenSize: parsedGardenSize,
|
||||
shortDescription: descriptions
|
||||
.first()
|
||||
.text()
|
||||
.trim(),
|
||||
longDescription: descriptions
|
||||
.last()
|
||||
.text()
|
||||
.trim(),
|
||||
streetNumber: 0,
|
||||
streetName: "",
|
||||
locality: "",
|
||||
municipality: "",
|
||||
city: "",
|
||||
region: "",
|
||||
entity: "",
|
||||
country: "",
|
||||
locationLat,
|
||||
locationLong,
|
||||
adStatus: status,
|
||||
publishedDate: publishedDateMoment.toISOString(),
|
||||
renewedDate: renewedDateMoment.toISOString(),
|
||||
numberOfRooms,
|
||||
numberOfFloors,
|
||||
floor,
|
||||
accessRoadType,
|
||||
heatingType,
|
||||
furnishingType,
|
||||
balcony,
|
||||
newBuilding,
|
||||
elevator,
|
||||
water,
|
||||
electricity,
|
||||
drainageSystem,
|
||||
registeredInZkBooks,
|
||||
recentlyAdapted,
|
||||
parking,
|
||||
garage,
|
||||
gas,
|
||||
antiTheftDoor,
|
||||
airCondition,
|
||||
phoneConnection,
|
||||
cableTV,
|
||||
internet,
|
||||
basementAttic,
|
||||
storeRoom,
|
||||
videoSurveillance,
|
||||
alarm,
|
||||
suitableForStudents,
|
||||
includingBills,
|
||||
animalsAllowed,
|
||||
pool,
|
||||
urbanPlanPermit,
|
||||
buildingPermit,
|
||||
utilityConnection,
|
||||
distanceToRiver,
|
||||
numberOfViewsAgency
|
||||
};
|
||||
//
|
||||
//console.log("Scraped data:", data);
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
console.error("Exception caught: " + e.message, "\r\nURL:", url);
|
||||
}
|
||||
} while (hasParseErrors && numberOfParseErrors <= 1);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user