Compare commits
1 Commits
prostor-vi
...
sliders-fo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6725355a0 |
@@ -7,7 +7,42 @@ const PRICE_SLIDER_OPTIONS_SALE = {
|
|||||||
step: 1000,
|
step: 1000,
|
||||||
connect: true
|
connect: true
|
||||||
};
|
};
|
||||||
|
const FLAT_PRICE_SLIDER_OPTIONS_SALE = {
|
||||||
|
start: [50000, 150000],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [800000]
|
||||||
|
},
|
||||||
|
step: 5000,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
|
const HOUSE_PRICE_SLIDER_OPTIONS_SALE = {
|
||||||
|
start: [50000, 150000],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [1500000]
|
||||||
|
},
|
||||||
|
step: 10000,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
|
const OFFICE_PRICE_SLIDER_OPTIONS_SALE = {
|
||||||
|
start: [15000, 50000],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [2000000]
|
||||||
|
},
|
||||||
|
step: 2000,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
|
const LAND_PRICE_SLIDER_OPTIONS_SALE = {
|
||||||
|
start: [40000, 80000],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [2000000]
|
||||||
|
},
|
||||||
|
step: 10000,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
const PRICE_SLIDER_OPTIONS_RENT = {
|
const PRICE_SLIDER_OPTIONS_RENT = {
|
||||||
start: [300, 500],
|
start: [300, 500],
|
||||||
range: {
|
range: {
|
||||||
@@ -17,18 +52,62 @@ const PRICE_SLIDER_OPTIONS_RENT = {
|
|||||||
step: 50,
|
step: 50,
|
||||||
connect: true
|
connect: true
|
||||||
};
|
};
|
||||||
|
const FLAT_PRICE_SLIDER_OPTIONS_RENT = {
|
||||||
|
start: [300, 600],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [4000]
|
||||||
|
},
|
||||||
|
step: 100,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
|
const HOUSE_PRICE_SLIDER_OPTIONS_RENT = {
|
||||||
|
start: [500, 1000],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [10000]
|
||||||
|
},
|
||||||
|
step: 100,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
|
const OFFICE_PRICE_SLIDER_OPTIONS_RENT = {
|
||||||
|
start: [200, 1000],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [20000]
|
||||||
|
},
|
||||||
|
step: 100,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
|
const LAND_PRICE_SLIDER_OPTIONS_RENT = {
|
||||||
|
start: [500, 1000],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [20000]
|
||||||
|
},
|
||||||
|
step: 100,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
//This will be used for Flats, Apartments, Houses
|
//This will be used for Flats, Apartments, Houses
|
||||||
const HOME_SIZE_SLIDER_OPTIONS = {
|
const HOME_SIZE_SLIDER_OPTIONS = {
|
||||||
start: [30, 75],
|
start: [30, 75],
|
||||||
range: {
|
range: {
|
||||||
min: [0],
|
min: [0],
|
||||||
max: [400]
|
max: [500]
|
||||||
},
|
},
|
||||||
step: 5,
|
step: 5,
|
||||||
connect: true
|
connect: true
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const OFFICE_SIZE_SLIDER_OPTIONS = {
|
||||||
|
start: [30, 150],
|
||||||
|
range: {
|
||||||
|
min: [0],
|
||||||
|
max: [1200]
|
||||||
|
},
|
||||||
|
step: 10,
|
||||||
|
connect: true
|
||||||
|
};
|
||||||
const GARDEN_SIZE_SLIDER_OPTIONS = {
|
const GARDEN_SIZE_SLIDER_OPTIONS = {
|
||||||
start: [100, 1000],
|
start: [100, 1000],
|
||||||
range: {
|
range: {
|
||||||
@@ -111,8 +190,8 @@ const AD_CATEGORY = {
|
|||||||
hasNumberOfRoom: true,
|
hasNumberOfRoom: true,
|
||||||
hasNumberOfFloors: false,
|
hasNumberOfFloors: false,
|
||||||
hasFloorProp: true,
|
hasFloorProp: true,
|
||||||
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
|
priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
|
||||||
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
|
priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
|
||||||
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
|
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
|
||||||
},
|
},
|
||||||
HOUSE: {
|
HOUSE: {
|
||||||
@@ -126,8 +205,8 @@ const AD_CATEGORY = {
|
|||||||
hasNumberOfRoom: true,
|
hasNumberOfRoom: true,
|
||||||
hasNumberOfFloors: true,
|
hasNumberOfFloors: true,
|
||||||
hasFloorProp: false,
|
hasFloorProp: false,
|
||||||
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
|
priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
|
||||||
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
|
priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
|
||||||
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
|
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
|
||||||
gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
|
gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
|
||||||
},
|
},
|
||||||
@@ -142,9 +221,9 @@ const AD_CATEGORY = {
|
|||||||
hasNumberOfRoom: true,
|
hasNumberOfRoom: true,
|
||||||
hasNumberOfFloors: false,
|
hasNumberOfFloors: false,
|
||||||
hasFloorProp: true,
|
hasFloorProp: true,
|
||||||
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
|
priceSliderOptionsSale: OFFICE_PRICE_SLIDER_OPTIONS_SALE,
|
||||||
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
|
priceSliderOptionsRent: OFFICE_PRICE_SLIDER_OPTIONS_RENT,
|
||||||
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
|
sizeSliderOptions: OFFICE_SIZE_SLIDER_OPTIONS
|
||||||
},
|
},
|
||||||
LAND: {
|
LAND: {
|
||||||
id: "LAND",
|
id: "LAND",
|
||||||
@@ -157,8 +236,8 @@ const AD_CATEGORY = {
|
|||||||
hasNumberOfRoom: false,
|
hasNumberOfRoom: false,
|
||||||
hasNumberOfFloors: false,
|
hasNumberOfFloors: false,
|
||||||
hasFloorProp: false,
|
hasFloorProp: false,
|
||||||
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
|
priceSliderOptionsSale: LAND_PRICE_SLIDER_OPTIONS_SALE,
|
||||||
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
|
priceSliderOptionsRent: LAND_PRICE_SLIDER_OPTIONS_RENT,
|
||||||
sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
|
sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
|
||||||
},
|
},
|
||||||
APARTMENT: {
|
APARTMENT: {
|
||||||
@@ -172,8 +251,8 @@ const AD_CATEGORY = {
|
|||||||
hasNumberOfRoom: true,
|
hasNumberOfRoom: true,
|
||||||
hasNumberOfFloors: false,
|
hasNumberOfFloors: false,
|
||||||
hasFloorProp: true,
|
hasFloorProp: true,
|
||||||
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
|
priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
|
||||||
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
|
priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
|
||||||
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
|
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
|
||||||
},
|
},
|
||||||
GARAGE: {
|
GARAGE: {
|
||||||
@@ -202,8 +281,8 @@ const AD_CATEGORY = {
|
|||||||
hasNumberOfRoom: true,
|
hasNumberOfRoom: true,
|
||||||
hasNumberOfFloors: true,
|
hasNumberOfFloors: true,
|
||||||
hasFloorProp: false,
|
hasFloorProp: false,
|
||||||
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
|
priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
|
||||||
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
|
priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
|
||||||
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
|
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
|
||||||
gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
|
gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
|
||||||
}
|
}
|
||||||
@@ -216,8 +295,7 @@ const AD_STATUS = {
|
|||||||
STATUS_DELETED: 4,
|
STATUS_DELETED: 4,
|
||||||
STATUS_URGENT: 5,
|
STATUS_URGENT: 5,
|
||||||
STATUS_DISCOUNTED: 6,
|
STATUS_DISCOUNTED: 6,
|
||||||
STATUS_RENTED: 7,
|
STATUS_RENTED: 7
|
||||||
STATUS_VIP: 8
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const AD_AGENCY = {
|
const AD_AGENCY = {
|
||||||
|
|||||||
@@ -32,11 +32,6 @@ const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
|
|||||||
|
|
||||||
const API_MAP_KEY = process.env.API_MAP_KEY || "";
|
const API_MAP_KEY = process.env.API_MAP_KEY || "";
|
||||||
|
|
||||||
const PROSTOR_LOGIN = {
|
|
||||||
EMAIL: process.env.PROSTOR_LOGIN_EMAIL,
|
|
||||||
PASSWORD: process.env.PROSTOR_LOGIN_PASS
|
|
||||||
};
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
APP_PORT,
|
APP_PORT,
|
||||||
APP_URL,
|
APP_URL,
|
||||||
@@ -47,6 +42,5 @@ module.exports = {
|
|||||||
MAX_REAL_ESTATES_IN_EMAIL,
|
MAX_REAL_ESTATES_IN_EMAIL,
|
||||||
MAX_REAL_ESTATES_IN_FIRST_EMAIL,
|
MAX_REAL_ESTATES_IN_FIRST_EMAIL,
|
||||||
PRINT_CRAWLER_DEBUG,
|
PRINT_CRAWLER_DEBUG,
|
||||||
API_MAP_KEY,
|
API_MAP_KEY
|
||||||
PROSTOR_LOGIN
|
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -2,14 +2,13 @@
|
|||||||
const {
|
const {
|
||||||
findRealEstatesForSearchRequest
|
findRealEstatesForSearchRequest
|
||||||
} = require("../helpers/db/searchRequestMatch");
|
} = require("../helpers/db/searchRequestMatch");
|
||||||
const { AD_STATUS } = require("../common/enums");
|
|
||||||
|
|
||||||
const getRealEstates = async (req, res) => {
|
const getRealEstates = async (req, res) => {
|
||||||
const searchRequestId = req.params["searchRequestId"] || "";
|
const searchRequestId = req.params["searchRequestId"] || "";
|
||||||
const realEstates = await findRealEstatesForSearchRequest(searchRequestId);
|
const realEstates = await findRealEstatesForSearchRequest(searchRequestId);
|
||||||
|
|
||||||
const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage";
|
const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage";
|
||||||
res.render("realEstates", { realEstates, title, AD_STATUS });
|
res.render("realEstates", { realEstates, title });
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
const { getRealEstateById } = require("../helpers/db/realEstate");
|
const { getRealEstateById } = require("../helpers/db/realEstate");
|
||||||
const { AD_STATUS } = require("../common/enums");
|
|
||||||
|
|
||||||
const getRedirect = async (req, res) => {
|
const getRedirect = async (req, res) => {
|
||||||
const id = req.params.id || null;
|
const id = req.params.id || null;
|
||||||
let error = false;
|
let error = false;
|
||||||
let redirectUrl = undefined;
|
let redirectUrl = undefined;
|
||||||
let vipAd = undefined;
|
|
||||||
if (!id) {
|
if (!id) {
|
||||||
error = true;
|
error = true;
|
||||||
} else {
|
} else {
|
||||||
@@ -15,7 +13,6 @@ const getRedirect = async (req, res) => {
|
|||||||
error = true;
|
error = true;
|
||||||
} else {
|
} else {
|
||||||
redirectUrl = realEstate.url;
|
redirectUrl = realEstate.url;
|
||||||
vipAd = realEstate.adStatus === AD_STATUS.STATUS_VIP;
|
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
error = true;
|
error = true;
|
||||||
@@ -27,7 +24,7 @@ const getRedirect = async (req, res) => {
|
|||||||
res.render("notFound", { title });
|
res.render("notFound", { title });
|
||||||
} else {
|
} else {
|
||||||
const title = "Preusmjeravanje";
|
const title = "Preusmjeravanje";
|
||||||
res.render("redirect", { title, redirectUrl, vipAd });
|
res.render("redirect", { title, redirectUrl });
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
const fetch = require("node-fetch");
|
const fetch = require("node-fetch");
|
||||||
const cheerio = require("cheerio");
|
const cheerio = require("cheerio");
|
||||||
const moment = require("moment-timezone");
|
const moment = require("moment-timezone");
|
||||||
const FormData = require("form-data");
|
|
||||||
|
|
||||||
const {
|
const {
|
||||||
AD_TYPE,
|
AD_TYPE,
|
||||||
@@ -17,8 +16,7 @@ const {
|
|||||||
|
|
||||||
const {
|
const {
|
||||||
PRINT_CRAWLER_DEBUG,
|
PRINT_CRAWLER_DEBUG,
|
||||||
DEFAULT_TIMEZONE,
|
DEFAULT_TIMEZONE
|
||||||
PROSTOR_LOGIN
|
|
||||||
} = require("../../config/appConfig");
|
} = require("../../config/appConfig");
|
||||||
const { PROSTOR_FORCE_CRAWL } = require("../specificConfigs/prostor");
|
const { PROSTOR_FORCE_CRAWL } = require("../specificConfigs/prostor");
|
||||||
|
|
||||||
@@ -62,16 +60,13 @@ class ProstorCrawler {
|
|||||||
|
|
||||||
async crawl() {
|
async crawl() {
|
||||||
const crawlAdCategories = this.crawlerAdCategories;
|
const crawlAdCategories = this.crawlerAdCategories;
|
||||||
//We need session cookie to use login privileges
|
|
||||||
const prostorCookie = await this.getCookies();
|
|
||||||
//New tag to check if crawler loged in
|
|
||||||
const login = await this.loginForScraping(PROSTOR_LOGIN, prostorCookie);
|
|
||||||
const newRealEstates = [];
|
const newRealEstates = [];
|
||||||
//Crawl only if login was successful
|
|
||||||
if (crawlAdCategories && login) {
|
if (crawlAdCategories) {
|
||||||
const indexGenerators = [];
|
const indexGenerators = [];
|
||||||
for (const adCategory of crawlAdCategories) {
|
for (const adCategory of crawlAdCategories) {
|
||||||
indexGenerators.push(this.categoryIndexer(adCategory, prostorCookie));
|
indexGenerators.push(this.categoryIndexer(adCategory));
|
||||||
}
|
}
|
||||||
|
|
||||||
let done = false;
|
let done = false;
|
||||||
@@ -124,14 +119,13 @@ class ProstorCrawler {
|
|||||||
return newRealEstates;
|
return newRealEstates;
|
||||||
}
|
}
|
||||||
|
|
||||||
async *categoryIndexer(adCategory, prostorCookie) {
|
async *categoryIndexer(adCategory) {
|
||||||
const urlAdTypePart = PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
|
const urlAdTypePart = PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
|
||||||
const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
|
const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
|
||||||
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
|
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
|
||||||
const urlPageToCrawl = `${this.baseUrl}?remove_sold=0${urlAdTypePart}${urlCategoryPart}`;
|
const urlPageToCrawl = `${this.baseUrl}?remove_sold=0${urlAdTypePart}${urlCategoryPart}`;
|
||||||
const listOfAllRealEstates = await this.extractRealEstates(
|
const listOfAllRealEstates = await this.extractRealEstates(
|
||||||
urlPageToCrawl,
|
urlPageToCrawl
|
||||||
prostorCookie
|
|
||||||
);
|
);
|
||||||
|
|
||||||
let elementToStartIndexFrom = 0;
|
let elementToStartIndexFrom = 0;
|
||||||
@@ -145,8 +139,7 @@ class ProstorCrawler {
|
|||||||
elementToStartIndexFrom += realEstatesForSinglePage.length;
|
elementToStartIndexFrom += realEstatesForSinglePage.length;
|
||||||
|
|
||||||
const singlePageResults = await this.indexSinglePage(
|
const singlePageResults = await this.indexSinglePage(
|
||||||
realEstatesForSinglePage,
|
realEstatesForSinglePage
|
||||||
prostorCookie
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const filteredSinglePageResults = singlePageResults.filter(
|
const filteredSinglePageResults = singlePageResults.filter(
|
||||||
@@ -170,10 +163,10 @@ class ProstorCrawler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async indexSinglePage(realEstatesList, prostorCookie) {
|
async indexSinglePage(realEstatesList) {
|
||||||
const asyncActions = [];
|
const asyncActions = [];
|
||||||
for (const realEstate of realEstatesList) {
|
for (const realEstate of realEstatesList) {
|
||||||
asyncActions.push(this.scrapeAd(realEstate, prostorCookie));
|
asyncActions.push(this.scrapeAd(realEstate));
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -187,25 +180,12 @@ class ProstorCrawler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async scrapeAd(realEstate, prostorCookie) {
|
async scrapeAd(realEstate) {
|
||||||
const { lat, lng, property_name, price, size, link, status } = realEstate;
|
const { lat, lng, property_name, price, size, link, status } = realEstate;
|
||||||
|
|
||||||
//Status information is given already in realestate list
|
|
||||||
//For VIP Ads status ='' canot be used, but no VIP ads are crawled
|
|
||||||
//We will make "fake" vip ad for RE that have size=55
|
|
||||||
//It is weird because yesterday it said 'VIP ponuda' ???
|
|
||||||
const adStatus =
|
|
||||||
size === "55"
|
|
||||||
? ProstorCrawler.getStatusId("VIP ponuda")
|
|
||||||
: ProstorCrawler.getStatusId(status);
|
|
||||||
|
|
||||||
const url = `https://prostor.ba${link}`;
|
const url = `https://prostor.ba${link}`;
|
||||||
|
|
||||||
// console.log("[PROSTOR] Scraping : ", url);
|
// console.log("[PROSTOR] Scraping : ", url);
|
||||||
try {
|
try {
|
||||||
const adPageSource = await fetch(url, {
|
const adPageSource = await fetch(url);
|
||||||
headers: { Cookie: prostorCookie }
|
|
||||||
});
|
|
||||||
const body = await adPageSource.text();
|
const body = await adPageSource.text();
|
||||||
const $ = cheerio.load(body);
|
const $ = cheerio.load(body);
|
||||||
|
|
||||||
@@ -350,6 +330,7 @@ class ProstorCrawler {
|
|||||||
furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
|
furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const adStatus = ProstorCrawler.getStatusId(status);
|
||||||
const title = property_name;
|
const title = property_name;
|
||||||
const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
|
const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
|
||||||
const parsedArea = parseFloat(size);
|
const parsedArea = parseFloat(size);
|
||||||
@@ -427,15 +408,13 @@ class ProstorCrawler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async extractRealEstates(url, prostorCookie) {
|
async extractRealEstates(url) {
|
||||||
if (PRINT_CRAWLER_DEBUG) {
|
if (PRINT_CRAWLER_DEBUG) {
|
||||||
console.log("[PROSTOR] Index page : ", url);
|
console.log("[PROSTOR] Index page : ", url);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetch(url, {
|
const res = await fetch(url);
|
||||||
headers: { Cookie: prostorCookie }
|
|
||||||
});
|
|
||||||
const body = await res.text();
|
const body = await res.text();
|
||||||
const $ = cheerio.load(body);
|
const $ = cheerio.load(body);
|
||||||
|
|
||||||
@@ -569,8 +548,6 @@ class ProstorCrawler {
|
|||||||
return AD_STATUS.STATUS_SOLD;
|
return AD_STATUS.STATUS_SOLD;
|
||||||
case "Iznajmljeno":
|
case "Iznajmljeno":
|
||||||
return AD_STATUS.STATUS_RENTED;
|
return AD_STATUS.STATUS_RENTED;
|
||||||
case "VIP ponuda":
|
|
||||||
return AD_STATUS.STATUS_VIP;
|
|
||||||
default:
|
default:
|
||||||
console.log("[PROSTOR] Unknown AD_STATUS : [", statusText, "]");
|
console.log("[PROSTOR] Unknown AD_STATUS : [", statusText, "]");
|
||||||
return AD_STATUS.STATUS_NORMAL;
|
return AD_STATUS.STATUS_NORMAL;
|
||||||
@@ -592,51 +569,6 @@ class ProstorCrawler {
|
|||||||
return savers[0].save(results);
|
return savers[0].save(results);
|
||||||
//so that we can use some sequelize options and information when data is inserted
|
//so that we can use some sequelize options and information when data is inserted
|
||||||
}
|
}
|
||||||
async loginForScraping(PROSTOR_LOGIN, prostorCookie) {
|
|
||||||
let formData = new FormData();
|
|
||||||
formData.append("email", PROSTOR_LOGIN.EMAIL);
|
|
||||||
formData.append("password", PROSTOR_LOGIN.PASSWORD);
|
|
||||||
|
|
||||||
return fetch("https://prostor.ba/moj-prostor/prijava", {
|
|
||||||
method: "POST",
|
|
||||||
body: formData,
|
|
||||||
headers: { Cookie: prostorCookie }
|
|
||||||
})
|
|
||||||
.then(page => {
|
|
||||||
return page.text();
|
|
||||||
})
|
|
||||||
.then(resp => {
|
|
||||||
const $ = cheerio.load(resp);
|
|
||||||
if (
|
|
||||||
$("h1")
|
|
||||||
.text()
|
|
||||||
.indexOf("Dobrodošli") !== -1
|
|
||||||
) {
|
|
||||||
console.log("[PROSTOR]: Crawler loged in!");
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
console.log("[PROSTOR]: Crawler login failed - wrong credentials!");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.catch(err => {
|
|
||||||
console.log("[PROSTOR]: Crawler login error ", err);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async getCookies() {
|
|
||||||
const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
|
|
||||||
headers: { Cookie: "" }
|
|
||||||
});
|
|
||||||
const raw = getResponse.headers.raw()["set-cookie"];
|
|
||||||
const cookie = raw
|
|
||||||
.map(datastring => {
|
|
||||||
const data = datastring.split(";");
|
|
||||||
const cookieData = data[0];
|
|
||||||
return cookieData;
|
|
||||||
})
|
|
||||||
.join(";");
|
|
||||||
return cookie;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = ProstorCrawler;
|
module.exports = ProstorCrawler;
|
||||||
|
|||||||
@@ -23,19 +23,16 @@ const generateRealEstateLinks = realEstates => {
|
|||||||
const generateNotificationEmail = (
|
const generateNotificationEmail = (
|
||||||
realEstates,
|
realEstates,
|
||||||
searchRequestId,
|
searchRequestId,
|
||||||
noAllRealEstates,
|
|
||||||
dailyNotification = false
|
dailyNotification = false
|
||||||
) => {
|
) => {
|
||||||
const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
|
const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
|
||||||
|
|
||||||
const realEstatesToShow = truncateList
|
const realEstatesToShow = truncateList
|
||||||
? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
|
? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
|
||||||
: realEstates;
|
: realEstates;
|
||||||
|
|
||||||
const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
|
const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
|
||||||
|
|
||||||
const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
|
const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
|
||||||
const moreRealEstates = `<div>Kompletan spisak nekretnina (${noAllRealEstates}) možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
|
const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
|
||||||
const emailFooter = generateEmailFooter(searchRequestId);
|
const emailFooter = generateEmailFooter(searchRequestId);
|
||||||
const asapMessageBody =
|
const asapMessageBody =
|
||||||
realEstates.length > 1
|
realEstates.length > 1
|
||||||
@@ -73,7 +70,6 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
|
|||||||
} = searchRequest;
|
} = searchRequest;
|
||||||
|
|
||||||
const realEstateLinks = generateRealEstateLinks(matchingRealEstates);
|
const realEstateLinks = generateRealEstateLinks(matchingRealEstates);
|
||||||
|
|
||||||
const instantRealEstatesText = `<br/>
|
const instantRealEstatesText = `<br/>
|
||||||
<div>
|
<div>
|
||||||
U međuvremenu pogledajte neke od nedavno objavljenih nekretnina koje odgovaraju Vašim uslovima pretrage :<br/>
|
U međuvremenu pogledajte neke od nedavno objavljenih nekretnina koje odgovaraju Vašim uslovima pretrage :<br/>
|
||||||
|
|||||||
@@ -154,7 +154,3 @@ h3 {
|
|||||||
margin-top: 2rem;
|
margin-top: 2rem;
|
||||||
margin-bottom: 1rem;
|
margin-bottom: 1rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.estates-link {
|
|
||||||
color: rgba(0, 0, 0, 0.87);
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -8,10 +8,7 @@ const {
|
|||||||
generateNewSearchRequestEmail,
|
generateNewSearchRequestEmail,
|
||||||
generateEmailSubject
|
generateEmailSubject
|
||||||
} = require("../helpers/emailContentGenerator");
|
} = require("../helpers/emailContentGenerator");
|
||||||
const {
|
const { findNotNotifiedMatches } = require("../helpers/db/searchRequestMatch");
|
||||||
findNotNotifiedMatches,
|
|
||||||
findRealEstatesForSearchRequest
|
|
||||||
} = require("../helpers/db/searchRequestMatch");
|
|
||||||
const { sendEmail } = require("../services/emailService");
|
const { sendEmail } = require("../services/emailService");
|
||||||
|
|
||||||
const notifyForNewRealEstates = async newRealEstates => {
|
const notifyForNewRealEstates = async newRealEstates => {
|
||||||
@@ -42,18 +39,10 @@ const notifyMatches = async (matches, dailyNotification = false) => {
|
|||||||
const { email, subscribed } = searchRequest;
|
const { email, subscribed } = searchRequest;
|
||||||
if (notifyNow && subscribed) {
|
if (notifyNow && subscribed) {
|
||||||
const allMatchingRealEstates = matches[id].realEstates || [];
|
const allMatchingRealEstates = matches[id].realEstates || [];
|
||||||
|
|
||||||
//Variable allMatchingRealEstates are real estates that are "new" on the market
|
|
||||||
//the ones that we notify user in this moment, not all that already exists in db
|
|
||||||
//New variable allRealEstates are all real estates that exists in db for search req
|
|
||||||
const allRealEstates = await findRealEstatesForSearchRequest(id);
|
|
||||||
const noAllRealEstates = allRealEstates.length;
|
|
||||||
|
|
||||||
if (allMatchingRealEstates.length > 0) {
|
if (allMatchingRealEstates.length > 0) {
|
||||||
const emailContent = generateNotificationEmail(
|
const emailContent = generateNotificationEmail(
|
||||||
allMatchingRealEstates,
|
allMatchingRealEstates,
|
||||||
id,
|
id,
|
||||||
noAllRealEstates,
|
|
||||||
dailyNotification
|
dailyNotification
|
||||||
);
|
);
|
||||||
const emailSubject = generateEmailSubject(
|
const emailSubject = generateEmailSubject(
|
||||||
|
|||||||
@@ -1,29 +1,13 @@
|
|||||||
<div class="row center-align">
|
<div class="row center-align">
|
||||||
<ul class="collection with-header">
|
<ul class="collection with-header">
|
||||||
<% for(const realEstate of realEstates) { %>
|
<% for(const realEstate of realEstates) { %>
|
||||||
<li class="collection-item">
|
<li class="collection-item">
|
||||||
<% if(realEstate.adStatus === AD_STATUS.STATUS_VIP) {%>
|
<div><%= realEstate.title %>
|
||||||
<div>
|
<a href="<%= realEstate.url %>" class="kivi-color secondary-content">
|
||||||
<% //This needs to do redirecting instead of direct link to realestate
|
|
||||||
%>
|
|
||||||
<a href="/redirect/<%= realEstate.id %>" class="estates-link">
|
|
||||||
<%= realEstate.title %>
|
|
||||||
<div class="kivi-color secondary-content">
|
|
||||||
<i class="material-icons">send</i>
|
<i class="material-icons">send</i>
|
||||||
</div>
|
</a>
|
||||||
</a>
|
</div>
|
||||||
</div>
|
</li>
|
||||||
<%} else { %>
|
<% } %>
|
||||||
<div>
|
</ul>
|
||||||
<a href="<%= realEstate.url %>" class="estates-link">
|
</div>
|
||||||
<%= realEstate.title %>
|
|
||||||
<div class="kivi-color secondary-content">
|
|
||||||
<i class="material-icons">send</i>
|
|
||||||
</div>
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
<% }%>
|
|
||||||
</li>
|
|
||||||
<% } %>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
|
|||||||
@@ -1,49 +1,26 @@
|
|||||||
<br /><br />
|
<br><br>
|
||||||
<div class="center">
|
<div class="center">
|
||||||
<div class="preloader-wrapper big active center">
|
<div class="preloader-wrapper big active center">
|
||||||
<div class="kivi-spinner-color spinner-layer spinner-green-only">
|
<div class="kivi-spinner-color spinner-layer spinner-green-only">
|
||||||
<div class="circle-clipper left">
|
<div class="circle-clipper left">
|
||||||
<div class="circle"></div>
|
<div class="circle"></div>
|
||||||
</div>
|
</div><div class="gap-patch">
|
||||||
<div class="gap-patch">
|
<div class="circle"></div>
|
||||||
<div class="circle"></div>
|
</div><div class="circle-clipper right">
|
||||||
</div>
|
<div class="circle"></div>
|
||||||
<div class="circle-clipper right">
|
</div>
|
||||||
<div class="circle"></div>
|
</div>
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<br />
|
<br>
|
||||||
<% if(vipAd) { %>
|
|
||||||
<div class="center">
|
<div class="center">
|
||||||
<h6>
|
<h6>
|
||||||
Ovaj oglas zahtijeva da budete član
|
<a href="<%= redirectUrl %>" rel="noreferrer" id="realEstateUrl">Kliknite ovdje ako Vas web preglednik ne preusmjeri automatski</a>
|
||||||
<a href="https://prostor.ba/" rel="noreferrer">Prostor.ba</a>.
|
</h6>
|
||||||
<br />
|
|
||||||
<br />
|
|
||||||
<a href="https://prostor.ba/moj-prostor/prijava" rel="noreferrer"
|
|
||||||
>Ulogujte se</a
|
|
||||||
>
|
|
||||||
ili napravite
|
|
||||||
<a href="https://prostor.ba/moj-prostor/registracija" rel="noreferrer"
|
|
||||||
>novi račun</a
|
|
||||||
>, a potom otvorite <a href="<%= redirectUrl %>" rel="noreferrer">oglas</a>.
|
|
||||||
</h6>
|
|
||||||
</div>
|
</div>
|
||||||
<% } else { %>
|
|
||||||
<div class="center">
|
|
||||||
<h6>
|
|
||||||
<a href="<%= redirectUrl %>" rel="noreferrer" id="realEstateUrl"
|
|
||||||
>Kliknite ovdje ako Vas web preglednik ne preusmjeri automatski</a
|
|
||||||
>
|
|
||||||
</h6>
|
|
||||||
</div>
|
|
||||||
<% }%>
|
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
window.onload = function() {
|
window.onload = function() {
|
||||||
document.getElementById("realEstateUrl").click();
|
document.getElementById('realEstateUrl').click();
|
||||||
};
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -51,8 +51,6 @@ PROSTOR_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories t
|
|||||||
PROSTOR_IGNORED_USERNAMES=!!! This is not used for prostor crawler !!!
|
PROSTOR_IGNORED_USERNAMES=!!! This is not used for prostor crawler !!!
|
||||||
PROSTOR_DELAY_BETWEEN_PAGES=!!! This is not used for prostor crawler !!!
|
PROSTOR_DELAY_BETWEEN_PAGES=!!! This is not used for prostor crawler !!!
|
||||||
PROSTOR_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
|
PROSTOR_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
|
||||||
PROSTOR_LOGIN_EMAIL=Email of valid Prostor.ba account for crawling purposes
|
|
||||||
PROSTOR_LOGIN_PASS=Password of valid Prostor.ba account for crawling purposes
|
|
||||||
#==AKTIDO==
|
#==AKTIDO==
|
||||||
AKTIDO_MAX_PAGES=Restrict crawler to this number of pages
|
AKTIDO_MAX_PAGES=Restrict crawler to this number of pages
|
||||||
AKTIDO_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
|
AKTIDO_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
|
||||||
|
|||||||
30
package-lock.json
generated
30
package-lock.json
generated
@@ -1346,23 +1346,13 @@
|
|||||||
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
|
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
|
||||||
},
|
},
|
||||||
"form-data": {
|
"form-data": {
|
||||||
"version": "3.0.0",
|
"version": "2.3.3",
|
||||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
|
||||||
"integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
|
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
|
||||||
"requires": {
|
"requires": {
|
||||||
"asynckit": "^0.4.0",
|
"asynckit": "^0.4.0",
|
||||||
"combined-stream": "^1.0.8",
|
"combined-stream": "^1.0.6",
|
||||||
"mime-types": "^2.1.12"
|
"mime-types": "^2.1.12"
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"combined-stream": {
|
|
||||||
"version": "1.0.8",
|
|
||||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
|
||||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
|
||||||
"requires": {
|
|
||||||
"delayed-stream": "~1.0.0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"forwarded": {
|
"forwarded": {
|
||||||
@@ -3440,18 +3430,6 @@
|
|||||||
"tough-cookie": "~2.4.3",
|
"tough-cookie": "~2.4.3",
|
||||||
"tunnel-agent": "^0.6.0",
|
"tunnel-agent": "^0.6.0",
|
||||||
"uuid": "^3.3.2"
|
"uuid": "^3.3.2"
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"form-data": {
|
|
||||||
"version": "2.3.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
|
|
||||||
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
|
|
||||||
"requires": {
|
|
||||||
"asynckit": "^0.4.0",
|
|
||||||
"combined-stream": "^1.0.6",
|
|
||||||
"mime-types": "^2.1.12"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"require-directory": {
|
"require-directory": {
|
||||||
|
|||||||
@@ -39,7 +39,6 @@
|
|||||||
"express": "^4.16.4",
|
"express": "^4.16.4",
|
||||||
"express-ejs-layouts": "^2.5.0",
|
"express-ejs-layouts": "^2.5.0",
|
||||||
"express-layout": "^0.1.0",
|
"express-layout": "^0.1.0",
|
||||||
"form-data": "^3.0.0",
|
|
||||||
"html-to-text": "^5.1.1",
|
"html-to-text": "^5.1.1",
|
||||||
"moment": "^2.24.0",
|
"moment": "^2.24.0",
|
||||||
"moment-timezone": "^0.5.26",
|
"moment-timezone": "^0.5.26",
|
||||||
|
|||||||
Reference in New Issue
Block a user