senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'saljic-crawler-fix' into 'master'

Browse Source 
Saljic crawler changed substring call.

See merge request saburly/marketalarm/web!95
This commit was merged in pull request #95.
This commit is contained in:
Naida Vatric
2020-02-26 15:09:22 +00:00
parent b913daa1f7 f56cd5b549
commit 44402a9cc4
1 changed files with 19 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
app/crawler/specificCrawlers/saljic.js
View File

@@ -218,7 +218,7 @@ class SaljicCrawler {
}
async scrapeAd(url, adType) {
// console.log("[SALJIC] Scraping : ", url);
console.log("[SALJIC] Scraping : ", url);
try {
const adPageSource = await fetch(url);
const body = await adPageSource.text();
@@ -227,7 +227,9 @@ class SaljicCrawler {
// No information for status ex. PRODAN
const status = AD_STATUS.STATUS_NORMAL;
//Extracting agency ID from url
const agencyObjectId = parseInt(url.substring(46, url.length));
const agencyObjectId = url
? parseInt(url.substring(46, url.length))
: null;
//Extracting main properties
const propertySelectors = {
@@ -272,14 +274,10 @@ class SaljicCrawler {
.trim();
const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
const latText = latAndLongSrc.substring(
latAndLongSrc.indexOf("marker=") + 7,
latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
);
const longText = latAndLongSrc.substring(
latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
latAndLongSrc.length
);
const tmpLatLong = latAndLongSrc.split("marker=")[1];
const latText = tmpLatLong.split("%2C")[0];
const longText = tmpLatLong.split("%2C")[1];
const locationLat = parseFloat(latText) || null;
const locationLong = parseFloat(longText) || null;
@@ -328,11 +326,10 @@ class SaljicCrawler {
let numberOfViewsKivi = null;
let streetNumber = 0;
let adStatus = status;
let shortDescription = descriptions.substring(
0,
descriptions.indexOf(".")
);
let longDescription = descriptions;
let shortDescription = descriptions
? descriptions.substring(0, descriptions.indexOf("."))
: "";
let longDescription = descriptions || "";
//Extracting data - Glavne karakteristike
let mainFieldIndex = 1;
do {
@@ -343,10 +340,14 @@ class SaljicCrawler {
.replace(/[\n\r\t]/gm, "")
.trim();
const mainFieldTitle = mainField.substring(0, mainField.indexOf(" "));
const mainFieldTitle = mainField
? mainField.substring(0, mainField.indexOf(" "))
: "";
const mainFieldValue = mainField
.substring(mainField.indexOf(" "), mainField.length)
.trim();
? mainField
.substring(mainField.indexOf(" "), mainField.length)
.trim()
: "";
switch (mainFieldTitle) {
case "Površina":