371 lines
11 KiB
JavaScript
371 lines
11 KiB
JavaScript
"use strict";
|
|
|
|
const fetch = require("node-fetch");
|
|
const cheerio = require("cheerio");
|
|
const Promise = require("bluebird");
|
|
const moment = require("moment-timezone");
|
|
const htmlToText = require("html-to-text");
|
|
|
|
const {
|
|
AD_TYPE,
|
|
AD_CATEGORY,
|
|
AD_AGENCY,
|
|
AD_STATUS,
|
|
CRAWLER_AD_TYPE
|
|
} = require("../../common/enums");
|
|
|
|
const {
|
|
DEFAULT_TIMEZONE,
|
|
PRINT_CRAWLER_DEBUG
|
|
} = require("../../config/appConfig");
|
|
|
|
const AKTIDO_ENUMS = {
|
|
AKTIDO_AD_TYPE: {
|
|
[CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
|
|
[CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
|
|
[CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
|
|
},
|
|
AKTIDO_AD_CATEGORY: {
|
|
[AD_CATEGORY.ALL.id]: "",
|
|
[AD_CATEGORY.FLAT.id]: "/tip-2",
|
|
[AD_CATEGORY.HOUSE.id]: "/tip-1",
|
|
[AD_CATEGORY.LAND.id]: "/tip-5",
|
|
[AD_CATEGORY.OFFICE.id]: "/tip-4",
|
|
[AD_CATEGORY.APARTMENT.id]: "/tip-3",
|
|
[AD_CATEGORY.GARAGE.id]: "/tip-6"
|
|
//[AD_CATEGORY.COTTAGE.id]: ""
|
|
},
|
|
AKTIDO_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
|
|
AKTIDO_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
|
|
};
|
|
|
|
const { AKTIDO_FORCE_CRAWL } = require("../specificConfigs/aktido");
|
|
|
|
class AktidoCrawler {
|
|
constructor(
|
|
savers = [],
|
|
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
|
|
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
|
|
maxPages = 1000,
|
|
maxResultsPerPage = 100,
|
|
ignoredUsernames = [],
|
|
delayBetweenPages = 1000
|
|
) {
|
|
this.savers = savers;
|
|
this.baseUrl = "https://www.aktido.ba/pretraga/sortiraj-date_DESC";
|
|
this.crawlerAdTypes = crawlerAdTypes;
|
|
this.crawlerAdCategories = crawlerAdCategories;
|
|
this.maxPages = maxPages;
|
|
this.maxResultsPerPage = maxResultsPerPage;
|
|
this.delayBetweenPages = delayBetweenPages;
|
|
}
|
|
|
|
async crawl() {
|
|
const crawlAdCategories = this.crawlerAdCategories;
|
|
|
|
const newRealEstates = [];
|
|
|
|
if (crawlAdCategories) {
|
|
const indexGenerators = [];
|
|
for (const adCategory of crawlAdCategories) {
|
|
indexGenerators.push(this.categoryIndexer(adCategory));
|
|
}
|
|
|
|
let done = false;
|
|
while (!done) {
|
|
const categoryIndexerPromises = [];
|
|
const generatorsToRemove = [];
|
|
for (const indexGenerator of indexGenerators) {
|
|
categoryIndexerPromises.push(indexGenerator.next());
|
|
generatorsToRemove.push(false);
|
|
}
|
|
|
|
const singlePageResults = await Promise.all(categoryIndexerPromises);
|
|
const entries = singlePageResults.entries();
|
|
|
|
for (const [index, { value: singlePageResult }] of entries) {
|
|
if (singlePageResult) {
|
|
const saveResults = await this.saveCrawledResults(singlePageResult);
|
|
const { newRecords } = saveResults;
|
|
|
|
newRealEstates.push(...newRecords);
|
|
|
|
if (
|
|
Array.isArray(newRecords) &&
|
|
newRecords.length === 0 &&
|
|
!AKTIDO_FORCE_CRAWL
|
|
) {
|
|
generatorsToRemove[index] = true;
|
|
}
|
|
} else {
|
|
//Generator returned undefined, remove this generator from array
|
|
generatorsToRemove[index] = true;
|
|
// console.log("Generator ", index + 1, "has no more pages");
|
|
}
|
|
}
|
|
|
|
// console.log("Generators state : ", generatorsToRemove);
|
|
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
|
|
if (generatorsToRemove[i]) {
|
|
// console.log("\tRemove generator ", i + 1);
|
|
indexGenerators.splice(i, 1);
|
|
}
|
|
}
|
|
if (indexGenerators.length === 0) {
|
|
done = true;
|
|
}
|
|
|
|
await this.sleep(this.delayBetweenPages);
|
|
}
|
|
}
|
|
return newRealEstates;
|
|
}
|
|
|
|
async *categoryIndexer(adCategory) {
|
|
let pageToIndex = 1;
|
|
|
|
const urlAdTypePart = AKTIDO_ENUMS.AKTIDO_AD_TYPE[this.crawlerAdTypes];
|
|
const urlCategoryPart = AKTIDO_ENUMS.AKTIDO_AD_CATEGORY[adCategory];
|
|
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
|
|
while (true) {
|
|
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
|
|
const singlePageResults = await this.indexSinglePage(
|
|
urlPageToCrawl,
|
|
this.maxResultsPerPage
|
|
);
|
|
|
|
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
|
|
yield singlePageResults;
|
|
} else {
|
|
return undefined;
|
|
}
|
|
|
|
++pageToIndex;
|
|
if (pageToIndex === this.maxPages) {
|
|
return undefined;
|
|
}
|
|
}
|
|
} else {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
async indexSinglePage(url, maxResultsPerPage) {
|
|
if (PRINT_CRAWLER_DEBUG) {
|
|
console.log("[AKTIDO] Index page : ", url);
|
|
}
|
|
|
|
try {
|
|
const res = await fetch(url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
let hrefs = [];
|
|
|
|
$(
|
|
"body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
|
|
)
|
|
.find(".moreInfo")
|
|
.each((i, elem) => {
|
|
const href = $(elem)
|
|
.find("a")
|
|
.first()
|
|
.attr("href");
|
|
if (href) {
|
|
hrefs.push(href);
|
|
}
|
|
});
|
|
|
|
let actualNoOfResults =
|
|
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
|
|
|
|
const asyncScraping = [];
|
|
for (let i = 0; i < actualNoOfResults; i++) {
|
|
asyncScraping.push(this.scrapeAd(hrefs[i]));
|
|
}
|
|
|
|
const scrapedData = await Promise.all(asyncScraping);
|
|
const filteredScrapedData = scrapedData.filter(adData => !!adData);
|
|
return filteredScrapedData;
|
|
} catch (e) {
|
|
console.error("[AKTIDO] Exception caught:" + e);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async scrapeAd(url) {
|
|
// console.log("[AKTIDO] Scraping : ", url);
|
|
try {
|
|
const adPageSource = await fetch(url);
|
|
const body = await adPageSource.text();
|
|
const $ = cheerio.load(body);
|
|
|
|
const mapElementParent = $(".box-map").parent();
|
|
const scriptElement = $("script", mapElementParent);
|
|
if (
|
|
scriptElement[0] &&
|
|
scriptElement[0].children &&
|
|
scriptElement[0].children[0] &&
|
|
scriptElement[0].children[0].data
|
|
) {
|
|
let extractedData;
|
|
try {
|
|
//data string starts with : var json_map_data = [{"r ...
|
|
//so we remove first 20 characters
|
|
|
|
const jsonData = scriptElement[0].children[0].data.substring(20);
|
|
const parsedJsonData = JSON.parse(jsonData);
|
|
extractedData = parsedJsonData[0];
|
|
} catch (e) {
|
|
throw { message: "Can't find ad data JSON" };
|
|
}
|
|
|
|
const aktidoId = extractedData["re_realEstates_id"];
|
|
const adCategory = this.getKiviCategoryIdFromAktidoId(
|
|
parseInt(extractedData["re_types_id"])
|
|
);
|
|
if (!adCategory) {
|
|
throw {
|
|
message: `Invalid category : ${extractedData["re_types_id"]}`
|
|
};
|
|
}
|
|
const adType = this.getKiviAdTypeFromAktidoActionId(
|
|
parseInt(extractedData["re_action_id"])
|
|
);
|
|
if (!adType) {
|
|
throw {
|
|
message: `Invalid ad type : ${extractedData["re_action_id"]}`
|
|
};
|
|
}
|
|
|
|
const title = extractedData["re_realEstates_portalName"];
|
|
const extractedPrice = parseFloat(
|
|
extractedData["re_realEstates_price"]
|
|
);
|
|
const price = extractedPrice ? extractedPrice : null;
|
|
const area = parseFloat(extractedData["re_realEstates_area"]);
|
|
const gardenSize = parseFloat(
|
|
extractedData["re_realEstates_fieldArea"]
|
|
);
|
|
const longDescription = htmlToText.fromString(
|
|
extractedData["re_realEstates_description"]
|
|
);
|
|
const locationLong = extractedData["re_realEstates_longitude"];
|
|
const locationLat = extractedData["re_realEstates_latitude"];
|
|
const publishedDateMoment = moment.tz(
|
|
extractedData["re_realEstates_inserted"],
|
|
AKTIDO_ENUMS.AKTIDO_PUBLISHED_DATE_FORMAT,
|
|
DEFAULT_TIMEZONE
|
|
);
|
|
if (!publishedDateMoment.isValid()) {
|
|
throw {
|
|
message: `Invalid published date : ${
|
|
extractedData["re_realEstates_inserted"]
|
|
}`
|
|
};
|
|
}
|
|
|
|
const renewedDateMoment = moment.tz(
|
|
extractedData["re_realEstates_edited"],
|
|
AKTIDO_ENUMS.AKTIDO_RENEWED_DATE_FORMAT,
|
|
DEFAULT_TIMEZONE
|
|
);
|
|
if (!renewedDateMoment.isValid()) {
|
|
throw {
|
|
message: `Invalid renewed date : ${
|
|
extractedData["re_realEstates_edited"]
|
|
}`
|
|
};
|
|
}
|
|
|
|
const adStatus = AD_STATUS.STATUS_NORMAL;
|
|
|
|
const data = {
|
|
url,
|
|
agencyObjectId: aktidoId,
|
|
originAgencyName: AD_AGENCY.AKTIDO,
|
|
realEstateType: adCategory,
|
|
adType,
|
|
title,
|
|
price,
|
|
area,
|
|
gardenSize,
|
|
shortDescription: "",
|
|
longDescription: longDescription,
|
|
streetNumber: 0,
|
|
streetName: "",
|
|
locality: "",
|
|
municipality: "",
|
|
city: "",
|
|
region: "",
|
|
entity: "",
|
|
country: "",
|
|
locationLat,
|
|
locationLong,
|
|
adStatus,
|
|
publishedDate: publishedDateMoment.toISOString(),
|
|
renewedDate: renewedDateMoment.toISOString()
|
|
};
|
|
|
|
return data;
|
|
} else {
|
|
console.log("[AKTIDO] No JSON data for this ad : ", url);
|
|
return null;
|
|
}
|
|
} catch (e) {
|
|
console.error("[AKTIDO] Exception caught: " + e.message, "\r\nURL:", url);
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
//======= HELPER FUNCTIONS =============
|
|
|
|
getKiviCategoryIdFromAktidoId(aktidoCategoryId) {
|
|
switch (aktidoCategoryId) {
|
|
case 1:
|
|
return AD_CATEGORY.HOUSE.id;
|
|
case 2:
|
|
return AD_CATEGORY.FLAT.id;
|
|
case 3:
|
|
return AD_CATEGORY.APARTMENT.id;
|
|
case 4:
|
|
return AD_CATEGORY.OFFICE.id;
|
|
case 5:
|
|
return AD_CATEGORY.LAND.id;
|
|
case 6:
|
|
return AD_CATEGORY.GARAGE.id;
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
getKiviAdTypeFromAktidoActionId(actionId) {
|
|
switch (actionId) {
|
|
case 1:
|
|
return AD_TYPE.AD_TYPE_SALE.stringId;
|
|
case 2:
|
|
return AD_TYPE.AD_TYPE_RENT.stringId;
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
async sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async saveCrawledResults(results) {
|
|
const savers = this.savers;
|
|
|
|
// for (const saver of savers) {
|
|
// await saver.save(results);
|
|
// }
|
|
|
|
//For now, we use only Postgres saver, so ...
|
|
return await savers[0].save(results);
|
|
//so that we can use some sequelize options and information when data is inserted
|
|
}
|
|
}
|
|
|
|
module.exports = AktidoCrawler;
|