551 lines
15 KiB
JavaScript
551 lines
15 KiB
JavaScript
"use strict";
|
|
|
|
const fetch = require("node-fetch");
|
|
const cheerio = require("cheerio");
|
|
const moment = require("moment-timezone");
|
|
|
|
const {
|
|
AD_TYPE,
|
|
AD_CATEGORY,
|
|
AD_AGENCY,
|
|
AD_STATUS,
|
|
CRAWLER_AD_TYPE
|
|
} = require("../../common/enums");
|
|
|
|
const {
|
|
PRINT_CRAWLER_DEBUG,
|
|
DEFAULT_TIMEZONE
|
|
} = require("../../config/appConfig");
|
|
const { PROSTOR_FORCE_CRAWL } = require("../specificConfigs/prostor");
|
|
|
|
const PROSTOR_ENUMS = {
|
|
PROSTOR_AD_TYPE: {
|
|
[CRAWLER_AD_TYPE.ALL]: "&action=0",
|
|
[CRAWLER_AD_TYPE.ONLY_SELL]: "&action=1",
|
|
[CRAWLER_AD_TYPE.ONLY_RENT]: "&action=2"
|
|
},
|
|
PROSTOR_AD_CATEGORY: {
|
|
[AD_CATEGORY.ALL.id]: "",
|
|
[AD_CATEGORY.FLAT.id]: "&type=7",
|
|
[AD_CATEGORY.HOUSE.id]: "&type=8",
|
|
[AD_CATEGORY.LAND.id]: "&type=10",
|
|
[AD_CATEGORY.OFFICE.id]: "&type=9",
|
|
[AD_CATEGORY.APARTMENT.id]: "&type=11",
|
|
[AD_CATEGORY.GARAGE.id]: "&type=14"
|
|
//[AD_CATEGORY.COTTAGE.id]: ""
|
|
},
|
|
PROSTOR_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
|
|
PROSTOR_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
|
|
};
|
|
|
|
class ProstorCrawler {
|
|
constructor(
|
|
savers = [],
|
|
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
|
|
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
|
|
maxPages = 5000,
|
|
maxResultsPerPage = 5000,
|
|
ignoredUsernames = [],
|
|
delayBetweenPages = 1000
|
|
) {
|
|
this.savers = savers;
|
|
this.baseUrl = "https://prostor.ba/pretraga";
|
|
this.crawlerAdTypes = crawlerAdTypes;
|
|
this.crawlerAdCategories = crawlerAdCategories;
|
|
this.maxResultsPerPage = maxResultsPerPage;
|
|
this.delayBetweenPages = delayBetweenPages;
|
|
}
|
|
|
|
async crawlOld() {
|
|
const crawlAdCategories = this.crawlerAdCategories;
|
|
const newRealEstates = [];
|
|
|
|
if (crawlAdCategories) {
|
|
for (const adCategory of crawlAdCategories) {
|
|
const urlAdTypePart =
|
|
PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
|
|
const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
|
|
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
|
|
const urlPageToCrawl = `${this.baseUrl}?remove_sold=1${urlAdTypePart}${urlCategoryPart}`;
|
|
const singleCategoryResults = await this.extractRealEstates(
|
|
urlPageToCrawl
|
|
);
|
|
|
|
const resultsSubset = singleCategoryResults.slice(
|
|
0,
|
|
this.maxResultsPerPage
|
|
);
|
|
|
|
const saveResults = await this.saveCrawledResults(resultsSubset);
|
|
const { newRecords } = saveResults;
|
|
newRealEstates.push(...newRecords);
|
|
}
|
|
}
|
|
}
|
|
return newRealEstates;
|
|
}
|
|
|
|
async crawl() {
|
|
const crawlAdCategories = this.crawlerAdCategories;
|
|
|
|
const newRealEstates = [];
|
|
|
|
if (crawlAdCategories) {
|
|
const indexGenerators = [];
|
|
for (const adCategory of crawlAdCategories) {
|
|
indexGenerators.push(this.categoryIndexer(adCategory));
|
|
}
|
|
|
|
let done = false;
|
|
while (!done) {
|
|
const categoryIndexerPromises = [];
|
|
const generatorsToRemove = [];
|
|
for (const indexGenerator of indexGenerators) {
|
|
categoryIndexerPromises.push(indexGenerator.next());
|
|
generatorsToRemove.push(false);
|
|
}
|
|
|
|
const singlePageResults = await Promise.all(categoryIndexerPromises);
|
|
const entries = singlePageResults.entries();
|
|
|
|
for (const [index, { value: singlePageResult }] of entries) {
|
|
if (singlePageResult) {
|
|
const saveResults = await this.saveCrawledResults(singlePageResult);
|
|
const { newRecords } = saveResults;
|
|
|
|
newRealEstates.push(...newRecords);
|
|
|
|
if (
|
|
Array.isArray(newRecords) &&
|
|
newRecords.length === 0 &&
|
|
!PROSTOR_FORCE_CRAWL
|
|
) {
|
|
generatorsToRemove[index] = true;
|
|
}
|
|
} else {
|
|
//Generator returned undefined, remove this generator from array
|
|
generatorsToRemove[index] = true;
|
|
// console.log("Generator ", index + 1, "has no more pages");
|
|
}
|
|
}
|
|
|
|
// console.log("Generators state : ", generatorsToRemove);
|
|
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
|
|
if (generatorsToRemove[i]) {
|
|
// console.log("\tRemove generator ", i + 1);
|
|
indexGenerators.splice(i, 1);
|
|
}
|
|
}
|
|
if (indexGenerators.length === 0) {
|
|
done = true;
|
|
}
|
|
|
|
await this.sleep(this.delayBetweenPages);
|
|
}
|
|
}
|
|
return newRealEstates;
|
|
}
|
|
|
|
async *categoryIndexer(adCategory) {
|
|
const urlAdTypePart = PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
|
|
const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
|
|
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
|
|
const urlPageToCrawl = `${this.baseUrl}?remove_sold=1${urlAdTypePart}${urlCategoryPart}`;
|
|
const listOfAllRealEstates = await this.extractRealEstates(
|
|
urlPageToCrawl
|
|
);
|
|
|
|
let elementToStartIndexFrom = 0;
|
|
while (true) {
|
|
const realEstatesForSinglePage = listOfAllRealEstates.slice(
|
|
elementToStartIndexFrom,
|
|
elementToStartIndexFrom + this.maxResultsPerPage
|
|
);
|
|
|
|
if (realEstatesForSinglePage.length > 0) {
|
|
elementToStartIndexFrom += realEstatesForSinglePage.length;
|
|
|
|
const singlePageResults = await this.indexSinglePage(
|
|
realEstatesForSinglePage
|
|
);
|
|
|
|
const filteredSinglePageResults = singlePageResults.filter(
|
|
singleResult => !!singleResult
|
|
);
|
|
|
|
if (
|
|
Array.isArray(filteredSinglePageResults) &&
|
|
filteredSinglePageResults.length > 0
|
|
) {
|
|
yield filteredSinglePageResults;
|
|
} else {
|
|
return undefined;
|
|
}
|
|
} else {
|
|
return undefined;
|
|
}
|
|
}
|
|
} else {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
async indexSinglePage(realEstatesList) {
|
|
const asyncActions = [];
|
|
for (const realEstate of realEstatesList) {
|
|
asyncActions.push(this.scrapeAd(realEstate));
|
|
}
|
|
|
|
try {
|
|
return await Promise.all(asyncActions);
|
|
} catch (e) {
|
|
console.log(
|
|
"[PROSTOR] Error crawling ads : ",
|
|
e.message || "UNKNOWN ERROR"
|
|
);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async scrapeAd(realEstate) {
|
|
const { lat, lng, property_name, price, size, link } = realEstate;
|
|
const url = `https://prostor.ba${link}`;
|
|
console.log("[PROSTOR] Scraping : ", url);
|
|
try {
|
|
const adPageSource = await fetch(url);
|
|
const body = await adPageSource.text();
|
|
const $ = cheerio.load(body);
|
|
|
|
let numberOfRooms = null,
|
|
numberOfFloors = null,
|
|
floor = null,
|
|
accessRoadType = null,
|
|
heatingType = null,
|
|
furnishingType = null,
|
|
balcony = null,
|
|
newBuilding = null,
|
|
elevator = null,
|
|
water = null,
|
|
electricity = null,
|
|
drainageSystem = null,
|
|
registeredInZkBooks = null,
|
|
recentlyAdapted = null,
|
|
parking = null,
|
|
garage = null,
|
|
gas = null,
|
|
antiTheftDoor = null,
|
|
airCondition = null,
|
|
phoneConnection = null,
|
|
cableTV = null,
|
|
internet = null,
|
|
basementAttic = null,
|
|
storeRoom = null,
|
|
videoSurveillance = null,
|
|
alarm = null,
|
|
suitableForStudents = null,
|
|
includingBills = null,
|
|
animalsAllowed = null,
|
|
pool = null,
|
|
urbanPlanPermit = null,
|
|
buildingPermit = null,
|
|
utilityConnection = null,
|
|
distanceToRiver = null,
|
|
numberOfViewsAgency = null;
|
|
|
|
// link contains part of the URL in the format of : /prodaja/stan/stup/9556
|
|
// general form is : /actionType/realEstateType/location/realEstateID
|
|
// linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
|
|
|
|
const linkParts = link.split("/");
|
|
|
|
const adType = ProstorCrawler.getAdTypeId(linkParts[1]);
|
|
const realEstateType = ProstorCrawler.getAdCategoryId(linkParts[2]);
|
|
const prostorId = linkParts[4];
|
|
|
|
if (!adType || !realEstateType || !prostorId) {
|
|
console.log(
|
|
"adType: ",
|
|
adType,
|
|
" reType: ",
|
|
realEstateType,
|
|
" prostorId: ",
|
|
prostorId,
|
|
"url: ",
|
|
url
|
|
);
|
|
return null;
|
|
}
|
|
|
|
const adStatus = AD_STATUS.STATUS_NORMAL;
|
|
const title = property_name;
|
|
const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
|
|
const parsedArea = parseFloat(size);
|
|
const gardenSize = null;
|
|
const longDescription = null;
|
|
const publishedDateMoment = moment.tz(DEFAULT_TIMEZONE);
|
|
if (!publishedDateMoment.isValid()) {
|
|
throw {
|
|
message: `Invalid published date`
|
|
};
|
|
}
|
|
|
|
const renewedDateMoment = moment.tz(DEFAULT_TIMEZONE);
|
|
if (!renewedDateMoment.isValid()) {
|
|
throw {
|
|
message: `Invalid renewed date`
|
|
};
|
|
}
|
|
|
|
const data = {
|
|
url,
|
|
agencyObjectId: prostorId,
|
|
originAgencyName: AD_AGENCY.PROSTOR,
|
|
realEstateType,
|
|
adType,
|
|
title,
|
|
price: parsedPrice,
|
|
area: parsedArea,
|
|
gardenSize,
|
|
shortDescription: "",
|
|
longDescription: longDescription,
|
|
streetNumber: 0,
|
|
streetName: "",
|
|
locality: "",
|
|
municipality: "",
|
|
city: "",
|
|
region: "",
|
|
entity: "",
|
|
country: "",
|
|
locationLat: lat,
|
|
locationLong: lng,
|
|
adStatus,
|
|
publishedDate: publishedDateMoment.toISOString(),
|
|
renewedDate: renewedDateMoment.toISOString(),
|
|
numberOfRooms,
|
|
numberOfFloors,
|
|
floor,
|
|
accessRoadType,
|
|
heatingType,
|
|
furnishingType,
|
|
balcony,
|
|
newBuilding,
|
|
elevator,
|
|
water,
|
|
electricity,
|
|
drainageSystem,
|
|
registeredInZkBooks,
|
|
recentlyAdapted,
|
|
parking,
|
|
garage,
|
|
gas,
|
|
antiTheftDoor,
|
|
airCondition,
|
|
phoneConnection,
|
|
cableTV,
|
|
internet,
|
|
basementAttic,
|
|
storeRoom,
|
|
videoSurveillance,
|
|
alarm,
|
|
suitableForStudents,
|
|
includingBills,
|
|
animalsAllowed,
|
|
pool,
|
|
urbanPlanPermit,
|
|
buildingPermit,
|
|
utilityConnection,
|
|
distanceToRiver,
|
|
numberOfViewsAgency
|
|
};
|
|
|
|
return data;
|
|
} catch (e) {
|
|
console.error(
|
|
"[PROSTOR] Exception caught: " + e.message,
|
|
"\r\nURL:",
|
|
url
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async extractRealEstates(url) {
|
|
if (PRINT_CRAWLER_DEBUG) {
|
|
console.log("[PROSTOR] Index page : ", url);
|
|
}
|
|
|
|
try {
|
|
const res = await fetch(url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
|
|
const scriptElement = $(
|
|
"body > div > div.container-fluid > script:nth-child(7)"
|
|
);
|
|
|
|
if (
|
|
scriptElement[0] &&
|
|
scriptElement[0].children &&
|
|
scriptElement[0].children[0] &&
|
|
scriptElement[0].children[0].data
|
|
) {
|
|
const scriptData = scriptElement[0].children[0].data;
|
|
|
|
try {
|
|
// script element data contains JS code and we need to extract only data for realEstates
|
|
// data string starts with : var map; var markers = [{"r ...
|
|
// so we remove first 23 characters
|
|
//
|
|
// real estate JSON data ends with ...}, ]; map = new...
|
|
// so we need to find index of that substring to know where to stop
|
|
// we will NOT include trailing comma because it breaks JSON parse, so we have to close ] bracket manually
|
|
|
|
const jsonEndIndex = scriptData.indexOf(", ]; map = new");
|
|
if (jsonEndIndex > -1) {
|
|
const jsonData = scriptData.substring(23, jsonEndIndex) + "]";
|
|
const realEstates = JSON.parse(jsonData);
|
|
|
|
// const transformedRealEstates = [];
|
|
//
|
|
// for (const realEstate of realEstates) {
|
|
// const transformedRealEstate = ProstorCrawler.transformRealEstateData(
|
|
// realEstate
|
|
// );
|
|
// if (transformedRealEstate) {
|
|
// transformedRealEstates.push(transformedRealEstate);
|
|
// }
|
|
// }
|
|
//
|
|
// return transformedRealEstates;
|
|
return realEstates;
|
|
} else {
|
|
throw {
|
|
message: "Something is wrong with JSON data or data is moved"
|
|
};
|
|
}
|
|
} catch (e) {
|
|
console.log(e);
|
|
throw e;
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.error(
|
|
"[PROSTOR] Exception caught:",
|
|
e.message || "UNKNOWN MESSAGE"
|
|
);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
static transformRealEstateData(realEstateData) {
|
|
try {
|
|
const { lat, lng, property_name, price, size, link } = realEstateData;
|
|
|
|
// link contains part of the URL in the format of : /prodaja/stan/stup/9556
|
|
// general form is : /actionType/realEstateType/location/realEstateID
|
|
// linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
|
|
|
|
const linkParts = link.split("/");
|
|
|
|
const adType = ProstorCrawler.getAdTypeId(linkParts[1]);
|
|
const realEstateType = ProstorCrawler.getAdCategoryId(linkParts[2]);
|
|
const prostorId = linkParts[4];
|
|
const url = `https://prostor.ba${link}`;
|
|
|
|
if (!adType || !realEstateType || !prostorId) {
|
|
return null;
|
|
}
|
|
|
|
const adStatus = AD_STATUS.STATUS_NORMAL;
|
|
const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
|
|
const parsedArea = parseFloat(size);
|
|
|
|
const data = {
|
|
url,
|
|
agencyObjectId: prostorId,
|
|
originAgencyName: AD_AGENCY.PROSTOR,
|
|
realEstateType,
|
|
adType,
|
|
title: property_name,
|
|
price: parsedPrice,
|
|
area: parsedArea,
|
|
gardenSize: null,
|
|
shortDescription: "",
|
|
longDescription: "",
|
|
streetNumber: 0,
|
|
streetName: "",
|
|
locality: "",
|
|
municipality: "",
|
|
city: "",
|
|
region: "",
|
|
entity: "",
|
|
country: "",
|
|
locationLat: lat,
|
|
locationLong: lng,
|
|
adStatus,
|
|
publishedDate: null,
|
|
renewedDate: null
|
|
};
|
|
|
|
return data;
|
|
} catch (e) {
|
|
console.error(
|
|
"[PROSTOR] Exception caught: " + e.message,
|
|
"\r\nURL:",
|
|
url
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
//======= HELPER FUNCTIONS =============
|
|
|
|
static getAdCategoryId(categoryText) {
|
|
switch (categoryText) {
|
|
case "stan":
|
|
return AD_CATEGORY.FLAT.id;
|
|
case "kuca":
|
|
return AD_CATEGORY.HOUSE.id;
|
|
case "apartman":
|
|
return AD_CATEGORY.APARTMENT.id;
|
|
case "poslovni-prostor":
|
|
return AD_CATEGORY.OFFICE.id;
|
|
case "garaza":
|
|
return AD_CATEGORY.GARAGE.id;
|
|
case "zemljiste":
|
|
return AD_CATEGORY.LAND.id;
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
static getAdTypeId(adTypeText) {
|
|
switch (adTypeText) {
|
|
case "prodaja":
|
|
return AD_TYPE.AD_TYPE_SALE.stringId;
|
|
case "najam":
|
|
return AD_TYPE.AD_TYPE_RENT.stringId;
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
async sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async saveCrawledResults(results) {
|
|
const savers = this.savers;
|
|
|
|
// for (const saver of savers) {
|
|
// await saver.save(results);
|
|
// }
|
|
|
|
//For now, we use only Postgres saver, so ...
|
|
return savers[0].save(results);
|
|
//so that we can use some sequelize options and information when data is inserted
|
|
}
|
|
}
|
|
|
|
module.exports = ProstorCrawler;
|