senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added delay between ads.

Browse Source
This commit is contained in:
Naida Vatric
2020-02-29 18:52:36 +01:00
parent 034106d87a
commit 97c09a6da1
3 changed files with 475 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/crawler/specificConfigs/olx.js
View File

@@ -35,5 +35,6 @@ module.exports = {
OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [], OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
OLX_DELAY_BETWEEN_PAGES: OLX_DELAY_BETWEEN_PAGES:
parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000, parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000,
OLX_DELAY_BETWEEN_ADS: parseInt(process.env.OLX_DELAY_BETWEEN_ADS) || 1000,
OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL) OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL)
}; };

942
app/crawler/specificCrawlers/olx.js
View File

@@ -42,7 +42,10 @@ const OLX_ENUMS = {
OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm" OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
}; };
const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx"); const {
OLX_FORCE_CRAWL,
OLX_DELAY_BETWEEN_ADS
} = require("../specificConfigs/olx");
class OlxCrawler { class OlxCrawler {
constructor( constructor(
@@ -52,7 +55,8 @@ class OlxCrawler {
maxPages = 1000, maxPages = 1000,
maxResultsPerPage = 100, maxResultsPerPage = 100,
ignoredUsernames = [], ignoredUsernames = [],
delayBetweenPages = 1000 delayBetweenPages = 1000,
delayBetweenAds = OLX_DELAY_BETWEEN_ADS
) { ) {
this.savers = savers; this.savers = savers;
this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum"; this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
@@ -62,6 +66,7 @@ class OlxCrawler {
this.maxResultsPerPage = maxResultsPerPage; this.maxResultsPerPage = maxResultsPerPage;
this.ignoredUsernames = ignoredUsernames; this.ignoredUsernames = ignoredUsernames;
this.delayBetweenPages = delayBetweenPages; this.delayBetweenPages = delayBetweenPages;
this.delayBetweenAds = delayBetweenAds;
} }
async crawl() { async crawl() {
@@ -193,6 +198,8 @@ class OlxCrawler {
const asyncScraping = []; const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) { for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefs[i])); asyncScraping.push(this.scrapeAd(hrefs[i]));
//Delaying next scrape ad request to avoid ScraperAPI server error
asyncScraping.push(this.sleep(this.delayBetweenAds));
} }
const scrapedData = await Promise.all(asyncScraping); const scrapedData = await Promise.all(asyncScraping);
@@ -206,492 +213,486 @@ class OlxCrawler {
async scrapeAd(url) { async scrapeAd(url) {
console.log("Scraping : ", url); console.log("Scraping : ", url);
let hasParseErrors = false; // let hasParseErrors = false;
let numberOfParseErrors = 0; //let numberOfParseErrors = 0;
do { // do {
try { try {
const adPageSource = await fetch(url); const adPageSource = await fetch(url);
const body = await adPageSource.text(); const body = await adPageSource.text();
const $ = cheerio.load(body); const $ = cheerio.load(body);
let status = AD_STATUS.STATUS_NORMAL; let status = AD_STATUS.STATUS_NORMAL;
//
console.log("Body:", body);
//
const propertySelectors = {
username:
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
title: "#naslovartikla",
descriptions: ".artikal_detaljniopis_tekst",
category:
"#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
};
const propertySelectors = { const username = $(propertySelectors.username)
username: .text()
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span", .trim();
title: "#naslovartikla", if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
descriptions: ".artikal_detaljniopis_tekst", return null;
category: }
"#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
};
const username = $(propertySelectors.username) const title = $(propertySelectors.title)
.text() .text()
.trim();
const descriptions = $(propertySelectors.descriptions);
const category = $(propertySelectors.category)
.text()
.trim();
//====== PRICE DETECTION AND EXTRACTION =====
let price = null;
let normalPrice = null;
let urgentPrice = null;
const normalPriceValue = $("#pc > p:nth-child(2)")
.text()
.trim();
const urgentPriceValue = $(
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
)
.text()
.trim();
//Debug
//console.log("Title:", title);
//console.log("Url scraped:", url);
// console.log("Normal price value:", normalPriceValue);
// console.log("Urgent price value:", urgentPriceValue);
//
if (normalPriceValue && normalPriceValue.length > 0) {
normalPrice = normalPriceValue
.replace(/\r\n|\n|\r/gm, "")
.replace("KM", "")
.trim(); .trim();
if (this.ignoredUsernames.includes((username || "").toLowerCase())) { if (
return null; $("#pc > p.n")
.text()
.indexOf("Hitna") !== -1
) {
status = AD_STATUS.STATUS_URGENT;
} else {
status = AD_STATUS.STATUS_NORMAL;
} }
} else {
const title = $(propertySelectors.title) throw { message: "Can't find normal price" };
.text() }
.trim(); if (urgentPriceValue && urgentPriceValue.length > 0) {
const descriptions = $(propertySelectors.descriptions); const priceValues = urgentPriceValue.replace("Cijena", "").split("KM");
const category = $(propertySelectors.category) //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
.text() if (priceValues.length > 0) {
.trim(); if (priceValues[0].trim().indexOf("Hitno") != -1) {
urgentPrice = priceValues[0].replace("Hitno", "").trim();
//====== PRICE DETECTION AND EXTRACTION =====
let price = null;
let normalPrice = null;
let urgentPrice = null;
const normalPriceValue = $("#pc > p:nth-child(2)")
.text()
.trim();
const urgentPriceValue = $(
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
)
.text()
.trim();
//Debug
//console.log("Title:", title);
//console.log("Url scraped:", url);
// console.log("Normal price value:", normalPriceValue);
// console.log("Urgent price value:", urgentPriceValue);
//
if (normalPriceValue && normalPriceValue.length > 0) {
normalPrice = normalPriceValue
.replace(/\r\n|\n|\r/gm, "")
.replace("KM", "")
.trim();
if (
$("#pc > p.n")
.text()
.indexOf("Hitna") !== -1
) {
status = AD_STATUS.STATUS_URGENT; status = AD_STATUS.STATUS_URGENT;
} else { } else {
status = AD_STATUS.STATUS_NORMAL; urgentPrice = priceValues[0].trim();
} }
} else { } else {
throw { message: "Can't find normal price" }; throw { message: "Can't find urgent price" };
}
if (urgentPriceValue && urgentPriceValue.length > 0) {
const priceValues = urgentPriceValue
.replace("Cijena", "")
.split("KM");
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
if (priceValues.length > 0) {
if (priceValues[0].trim().indexOf("Hitno") != -1) {
urgentPrice = priceValues[0].replace("Hitno", "").trim();
status = AD_STATUS.STATUS_URGENT;
} else {
urgentPrice = priceValues[0].trim();
}
} else {
throw { message: "Can't find urgent price" };
}
} }
}
price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice; price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice;
//====== OTHER AD INFORMATION =============== //====== OTHER AD INFORMATION ===============
let adType = null; let adType = null;
let olxId = null; let olxId = null;
let numberOfViewsAgency = null; let numberOfViewsAgency = null;
let otherInformationDivId; let otherInformationDivId;
//We need to locate DIV ID where other information are stored //We need to locate DIV ID where other information are stored
for (let possibleId = 1; possibleId <= 30; possibleId++) { for (let possibleId = 1; possibleId <= 30; possibleId++) {
const adTypeFieldTitle = $( const adTypeFieldTitle = $(
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1` `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
)
.text()
.trim();
if (adTypeFieldTitle === "Vrsta oglasa") {
otherInformationDivId = possibleId;
break;
}
}
if (!otherInformationDivId) {
throw { message: "Other information DIV could not be found" };
}
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
const publishedDate = $(publishedDateValueSelector)
.text()
.trim();
const publishedDateMoment = moment.tz(
publishedDate,
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!publishedDateMoment.isValid()) {
throw { message: "Invalid published date ! Check parsing format" };
}
const renewedDate = $(renewedDateFullValueSelector)
.data("content")
.trim();
const renewedDateMoment = moment.tz(
renewedDate,
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!renewedDateMoment) {
throw {
message:
"Invalid renewed date ! Check how parser parsed renewed date text"
};
}
adType = $(
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
) )
.text() .text()
.trim(); .trim();
const parsedCategory = this.getAdCategoryId(category); if (adTypeFieldTitle === "Vrsta oglasa") {
if (!parsedCategory) { otherInformationDivId = possibleId;
throw { message: `Unknown ad category [${category}]` }; break;
} }
const parsedAdType = this.getAdTypeId(adType);
if (!parsedAdType) {
throw { message: "Unknown ad type" };
}
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
.text()
.trim();
olxId = $(`${olxIdFieldSelector} > div.df2`)
.text()
.trim();
numberOfViewsAgency = parseInt(
$(numberOfViewsAgencyValueSelector)
.text()
.trim()
);
if (olxIdFieldTitle !== "OLX ID") {
throw { message: "Cannot find correct OLX ID" };
}
//===========================================
//====== DETAIL INFORMATION FIELDS ==========
let area,
gardenSize,
numberOfRooms = null,
numberOfFloors = null,
floor = null,
accessRoadType = null,
heatingType = null,
furnishingType = null,
balcony = null,
newBuilding = null,
elevator = null,
water = null,
electricity = null,
drainageSystem = null,
registeredInZkBooks = null,
recentlyAdapted = null,
parking = null,
garage = null,
gas = null,
antiTheftDoor = null,
airCondition = null,
phoneConnection = null,
cableTV = null,
internet = null,
basementAttic = null,
storeRoom = null,
videoSurveillance = null,
alarm = null,
suitableForStudents = null,
includingBills = null,
animalsAllowed = null,
pool = null,
urbanPlanPermit = null,
buildingPermit = null,
utilityConnection = null,
distanceToRiver = null;
let fieldIndex = 1;
do {
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
const fieldTitleSelector = `${fieldSelector} > div.df1`;
const fieldValueSelector = `${fieldSelector} > div.df2`;
const fieldTitle = $(fieldTitleSelector)
.text()
.trim()
.toLowerCase();
const fieldValue = $(fieldValueSelector)
.text()
.trim()
.toLowerCase();
switch (fieldTitle) {
case "kvadrata":
area = fieldValue;
break;
case "okućnica (kvadratura)":
gardenSize = fieldValue;
break;
case "broj soba":
numberOfRooms = this.parseNumberOfRooms(
fieldValue,
parsedCategory
);
break;
case "broj prostorija":
numberOfRooms = this.parseNumberOfRooms(
fieldValue,
parsedCategory
);
break;
case "broj spratova":
numberOfFloors = this.parseNumberOfFloors(
fieldValue,
parsedCategory
);
break;
case "sprat":
floor = this.parseFloorNumber(fieldValue, parsedCategory);
break;
case "vrsta grijanja":
heatingType = this.getHeatingTypeId(fieldValue);
break;
case "namješten?":
furnishingType = this.getFurnishingTypeId(fieldValue);
break;
case "namješten":
furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case "namještena":
furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case "voda":
water = true;
break;
case "struja":
electricity = true;
break;
case "kanalizacija":
drainageSystem = fieldValue !== "nema";
break;
case "godina izgradnje":
newBuilding = newBuilding || fieldValue === "novogradnja";
break;
case "kućni ljubimci":
animalsAllowed = fieldValue === "da";
break;
case "uknjiženo / zk":
registeredInZkBooks = true;
break;
case "uknjiženo (zk)":
registeredInZkBooks = true;
break;
case "novogradnja":
newBuilding = true;
break;
case "nedavno adaptiran":
recentlyAdapted = true;
break;
case "nedavno adaptirana":
recentlyAdapted = true;
break;
case "balkon":
balcony = true;
break;
case "lift":
elevator = true;
break;
case "parking":
parking = true;
break;
case "garaža":
garage = true;
break;
case "plin":
gas = true;
break;
case "blindirana vrata":
antiTheftDoor = true;
break;
case "klima":
airCondition = true;
break;
case "telefonski priključak":
phoneConnection = true;
break;
case "kablovska tv":
cableTV = true;
break;
case "internet":
internet = true;
break;
case "podrum/tavan":
basementAttic = true;
break;
case "ostava/špajz":
storeRoom = true;
break;
case "video nadzor":
videoSurveillance = true;
break;
case "alarm":
alarm = true;
break;
case "za studente":
suitableForStudents = true;
break;
case "uključen trošak režija":
includingBills = true;
break;
case "građevinska dozvola":
buildingPermit = true;
break;
case "komunalni priključak":
utilityConnection = true;
break;
case "urbanistička dozvola":
urbanPlanPermit = true;
break;
case "udaljenost od rijeke (m)":
distanceToRiver = parseInt(fieldValue) || null;
break;
case "prilaz":
accessRoadType = this.getAccessRoadTypeId(fieldValue);
break;
case "bazen":
pool = true;
break;
case "iznajmljeno":
status = AD_STATUS.STATUS_RENTED;
break;
default:
// console.log(fieldTitle, " = ", fieldValue);
break;
}
if (
++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS ||
fieldTitle === ""
) {
break;
}
} while (true);
//===========================================
//=========================================
const parsedArea = this.parseArea(area) || null;
const parsedGardenSize = this.parseArea(gardenSize) || null;
const parsedPrice = this.parsePrice(price) || null;
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
const locationLatLngMatches = latLngRegex.exec(body);
let locationLat = null;
let locationLong = null;
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
locationLat = parseFloat(locationLatLngMatches[1]) || null;
locationLong = parseFloat(locationLatLngMatches[2]) || null;
}
if (
title.indexOf("[PRODANO]") !== -1 ||
title.indexOf("[ZAVRŠENO]") !== -1
) {
status = AD_STATUS.STATUS_SOLD;
}
const data = {
url,
agencyObjectId: olxId,
originAgencyName: AD_AGENCY.OLX,
realEstateType: parsedCategory,
adType: parsedAdType,
title,
price: parsedPrice,
area: parsedArea,
gardenSize: parsedGardenSize,
shortDescription: descriptions
.first()
.text()
.trim(),
longDescription: descriptions
.last()
.text()
.trim(),
streetNumber: 0,
streetName: "",
locality: "",
municipality: "",
city: "",
region: "",
entity: "",
country: "",
locationLat,
locationLong,
adStatus: status,
publishedDate: publishedDateMoment.toISOString(),
renewedDate: renewedDateMoment.toISOString(),
numberOfRooms,
numberOfFloors,
floor,
accessRoadType,
heatingType,
furnishingType,
balcony,
newBuilding,
elevator,
water,
electricity,
drainageSystem,
registeredInZkBooks,
recentlyAdapted,
parking,
garage,
gas,
antiTheftDoor,
airCondition,
phoneConnection,
cableTV,
internet,
basementAttic,
storeRoom,
videoSurveillance,
alarm,
suitableForStudents,
includingBills,
animalsAllowed,
pool,
urbanPlanPermit,
buildingPermit,
utilityConnection,
distanceToRiver,
numberOfViewsAgency
};
//
//console.log("Scraped data:", data);
return data;
} catch (e) {
hasParseErrors = true;
numberOfParseErrors++;
console.error("Exception caught: " + e.message, "\r\nURL:", url);
} }
} while (hasParseErrors && numberOfParseErrors <= 1);
if (!otherInformationDivId) {
throw { message: "Other information DIV could not be found" };
}
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
const publishedDate = $(publishedDateValueSelector)
.text()
.trim();
const publishedDateMoment = moment.tz(
publishedDate,
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!publishedDateMoment.isValid()) {
throw { message: "Invalid published date ! Check parsing format" };
}
const renewedDate = $(renewedDateFullValueSelector)
.data("content")
.trim();
const renewedDateMoment = moment.tz(
renewedDate,
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!renewedDateMoment) {
throw {
message:
"Invalid renewed date ! Check how parser parsed renewed date text"
};
}
adType = $(
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
)
.text()
.trim();
const parsedCategory = this.getAdCategoryId(category);
if (!parsedCategory) {
throw { message: `Unknown ad category [${category}]` };
}
const parsedAdType = this.getAdTypeId(adType);
if (!parsedAdType) {
throw { message: "Unknown ad type" };
}
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
.text()
.trim();
olxId = $(`${olxIdFieldSelector} > div.df2`)
.text()
.trim();
numberOfViewsAgency = parseInt(
$(numberOfViewsAgencyValueSelector)
.text()
.trim()
);
if (olxIdFieldTitle !== "OLX ID") {
throw { message: "Cannot find correct OLX ID" };
}
//===========================================
//====== DETAIL INFORMATION FIELDS ==========
let area,
gardenSize,
numberOfRooms = null,
numberOfFloors = null,
floor = null,
accessRoadType = null,
heatingType = null,
furnishingType = null,
balcony = null,
newBuilding = null,
elevator = null,
water = null,
electricity = null,
drainageSystem = null,
registeredInZkBooks = null,
recentlyAdapted = null,
parking = null,
garage = null,
gas = null,
antiTheftDoor = null,
airCondition = null,
phoneConnection = null,
cableTV = null,
internet = null,
basementAttic = null,
storeRoom = null,
videoSurveillance = null,
alarm = null,
suitableForStudents = null,
includingBills = null,
animalsAllowed = null,
pool = null,
urbanPlanPermit = null,
buildingPermit = null,
utilityConnection = null,
distanceToRiver = null;
let fieldIndex = 1;
do {
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
const fieldTitleSelector = `${fieldSelector} > div.df1`;
const fieldValueSelector = `${fieldSelector} > div.df2`;
const fieldTitle = $(fieldTitleSelector)
.text()
.trim()
.toLowerCase();
const fieldValue = $(fieldValueSelector)
.text()
.trim()
.toLowerCase();
switch (fieldTitle) {
case "kvadrata":
area = fieldValue;
break;
case "okućnica (kvadratura)":
gardenSize = fieldValue;
break;
case "broj soba":
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
break;
case "broj prostorija":
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
break;
case "broj spratova":
numberOfFloors = this.parseNumberOfFloors(
fieldValue,
parsedCategory
);
break;
case "sprat":
floor = this.parseFloorNumber(fieldValue, parsedCategory);
break;
case "vrsta grijanja":
heatingType = this.getHeatingTypeId(fieldValue);
break;
case "namješten?":
furnishingType = this.getFurnishingTypeId(fieldValue);
break;
case "namješten":
furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case "namještena":
furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case "voda":
water = true;
break;
case "struja":
electricity = true;
break;
case "kanalizacija":
drainageSystem = fieldValue !== "nema";
break;
case "godina izgradnje":
newBuilding = newBuilding || fieldValue === "novogradnja";
break;
case "kućni ljubimci":
animalsAllowed = fieldValue === "da";
break;
case "uknjiženo / zk":
registeredInZkBooks = true;
break;
case "uknjiženo (zk)":
registeredInZkBooks = true;
break;
case "novogradnja":
newBuilding = true;
break;
case "nedavno adaptiran":
recentlyAdapted = true;
break;
case "nedavno adaptirana":
recentlyAdapted = true;
break;
case "balkon":
balcony = true;
break;
case "lift":
elevator = true;
break;
case "parking":
parking = true;
break;
case "garaža":
garage = true;
break;
case "plin":
gas = true;
break;
case "blindirana vrata":
antiTheftDoor = true;
break;
case "klima":
airCondition = true;
break;
case "telefonski priključak":
phoneConnection = true;
break;
case "kablovska tv":
cableTV = true;
break;
case "internet":
internet = true;
break;
case "podrum/tavan":
basementAttic = true;
break;
case "ostava/špajz":
storeRoom = true;
break;
case "video nadzor":
videoSurveillance = true;
break;
case "alarm":
alarm = true;
break;
case "za studente":
suitableForStudents = true;
break;
case "uključen trošak režija":
includingBills = true;
break;
case "građevinska dozvola":
buildingPermit = true;
break;
case "komunalni priključak":
utilityConnection = true;
break;
case "urbanistička dozvola":
urbanPlanPermit = true;
break;
case "udaljenost od rijeke (m)":
distanceToRiver = parseInt(fieldValue) || null;
break;
case "prilaz":
accessRoadType = this.getAccessRoadTypeId(fieldValue);
break;
case "bazen":
pool = true;
break;
case "iznajmljeno":
status = AD_STATUS.STATUS_RENTED;
break;
default:
// console.log(fieldTitle, " = ", fieldValue);
break;
}
if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
break;
}
} while (true);
//===========================================
//=========================================
const parsedArea = this.parseArea(area) || null;
const parsedGardenSize = this.parseArea(gardenSize) || null;
const parsedPrice = this.parsePrice(price) || null;
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
const locationLatLngMatches = latLngRegex.exec(body);
let locationLat = null;
let locationLong = null;
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
locationLat = parseFloat(locationLatLngMatches[1]) || null;
locationLong = parseFloat(locationLatLngMatches[2]) || null;
}
if (
title.indexOf("[PRODANO]") !== -1 ||
title.indexOf("[ZAVRŠENO]") !== -1
) {
status = AD_STATUS.STATUS_SOLD;
}
const data = {
url,
agencyObjectId: olxId,
originAgencyName: AD_AGENCY.OLX,
realEstateType: parsedCategory,
adType: parsedAdType,
title,
price: parsedPrice,
area: parsedArea,
gardenSize: parsedGardenSize,
shortDescription: descriptions
.first()
.text()
.trim(),
longDescription: descriptions
.last()
.text()
.trim(),
streetNumber: 0,
streetName: "",
locality: "",
municipality: "",
city: "",
region: "",
entity: "",
country: "",
locationLat,
locationLong,
adStatus: status,
publishedDate: publishedDateMoment.toISOString(),
renewedDate: renewedDateMoment.toISOString(),
numberOfRooms,
numberOfFloors,
floor,
accessRoadType,
heatingType,
furnishingType,
balcony,
newBuilding,
elevator,
water,
electricity,
drainageSystem,
registeredInZkBooks,
recentlyAdapted,
parking,
garage,
gas,
antiTheftDoor,
airCondition,
phoneConnection,
cableTV,
internet,
basementAttic,
storeRoom,
videoSurveillance,
alarm,
suitableForStudents,
includingBills,
animalsAllowed,
pool,
urbanPlanPermit,
buildingPermit,
utilityConnection,
distanceToRiver,
numberOfViewsAgency
};
//
//console.log("Scraped data:", data);
//Delay between real estate ads to avoid error from Scraper API
// await this.sleep(this.delayBetweenAds);
return data;
} catch (e) {
// hasParseErrors = true;
// numberOfParseErrors++;
console.error("Exception caught: " + e.message, "\r\nURL:", url);
}
// } while (hasParseErrors && numberOfParseErrors <= 1);
return null; return null;
} }
@@ -909,6 +910,7 @@ class OlxCrawler {
} }
async sleep(ms) { async sleep(ms) {
// console.log("Sleep for:", ms);
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise(resolve => setTimeout(resolve, ms));
} }

2
development.env
View File

@@ -43,7 +43,9 @@ OLX_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check commo
OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
OLX_DELAY_BETWEEN_ADS = time in miliseconds to wait before scraping next add to awoid server errors with Scraper API
OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
#==RENTAL== #==RENTAL==
RENTAL_MAX_PAGES=Restrict crawler to this number of pages RENTAL_MAX_PAGES=Restrict crawler to this number of pages
RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved