Compare commits

..

3 Commits

Author SHA1 Message Date
Senad Uka
e356f7a177 Added migrations 2019-03-10 05:23:52 +01:00
Senad Uka
055ca54002 Sendnotification moved to the lambda / has bugs 2019-03-08 06:24:10 +01:00
Senad Uka
3f8e6438b2 Serverless handler stub created 2019-03-07 06:24:08 +01:00
215 changed files with 23564 additions and 15501 deletions

5
.gitignore vendored
View File

@@ -1,5 +0,0 @@
node_modules/
.env
.idea/
.eslintrc
.vscode/

View File

@@ -1,12 +0,0 @@
FROM postgres:11.3
ENV POSTGIS_MAJOR 2.4
RUN apt-get update \
&& apt-get --assume-yes install postgresql-11-postgis-2.5-scripts\
&& apt-get --assume-yes install software-properties-common postgis\
&& rm -rf /var/lib/apt/lists/
RUN mkdir -p /docker-entrypoint-initdb.d
CMD ["postgres"]

View File

@@ -1 +0,0 @@
web: node ./index.js

View File

@@ -1,46 +1,4 @@
# MarketAlert
The purpose of this project is to build a web application that enables subscribing to notifications when new products are published on various ad based marketplaces. The MVP will be only based on OLX.ba
## Setup
* Before setup please confirm that Docker is installed `docker --version`. If not install it from official site.
### Setup with npm commands
1. Install packages
`npm install`
2. Run setup script
`npm run setup`
this will create and run postgres image and then execute migrations
3. Run app
`npm start` to run app without restart on changes or
`npm run start-mon` to run app with automatic restart on code change
# web
### Manual setup
1. Create postgres docker image
`docker build -t marketalerts .`
2. Run postgres image with
`docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts`
3. Install packages
`npm install`
4. Run migrations from `app` folder
`npm run migrate` or `npx sequelize db:migrate`
5. Run app
`npm start` or `npm run start-mon` to run app with automatic restart on code change
### AWS SES
- AWS SES credentials are handled with env vratiables
- Notification emails are sent in batches of 50, by using SES templates
- Make sure that you are using different templates for different envirorments
The purpose of this project is to build a web application that enables subscribing to notifications when new products are published on various ad based marketplaces. The MVP will be only based on OLX.ba

View File

@@ -1,342 +0,0 @@
const PRICE_SLIDER_OPTIONS_SALE = {
start: [50000, 85000],
range: {
min: [0],
max: [300000]
},
step: 1000,
connect: true
};
const PRICE_SLIDER_OPTIONS_RENT = {
start: [300, 500],
range: {
min: [0],
max: [2000]
},
step: 50,
connect: true
};
//This will be used for Flats, Apartments, Houses
const HOME_SIZE_SLIDER_OPTIONS = {
start: [30, 75],
range: {
min: [0],
max: [400]
},
step: 5,
connect: true
};
const GARDEN_SIZE_SLIDER_OPTIONS = {
start: [100, 1000],
range: {
min: [0],
max: [10000]
},
step: 100,
connect: true
};
const LAND_SIZE_SLIDER_OPTIONS = {
start: [5000, 15000],
range: {
min: [0],
max: [100000]
},
step: 100,
connect: true
};
const GARAGE_SIZE_SLIDER_OPTIONS = {
start: [10, 20],
range: {
min: [0],
max: [150]
},
step: 2,
connect: true
};
const GARAGE_PRICE_SLIDER_OPTIONS_SALE = {
start: [2000, 10000],
range: {
min: [0],
max: [60000]
},
step: 200,
connect: true
};
const GARAGE_PRICE_SLIDER_OPTIONS_RENT = {
start: [50, 150],
range: {
min: [0],
max: [1000]
},
step: 10,
connect: true
};
const AD_TYPE = {
AD_TYPE_SALE: {
id: 1,
stringId: "SALE",
title: "Kupi"
},
AD_TYPE_RENT: {
id: 2,
stringId: "RENT",
title: "Unajmi"
},
AD_TYPE_REQUEST: {
id: 3,
stringId: "REQUEST",
title: "Potražnja"
}
};
const AD_CATEGORY = {
ALL: {
id: "ALL"
},
FLAT: {
id: "FLAT",
title: "Stan",
hasGardenSize: false,
hasAccesRoadType: true,
hasBalconyProp: true,
hasNewBuildingProp: true,
hasElevatorProp: true,
hasNumberOfRoom: true,
hasNumberOfFloors: false,
hasFloorProp: true,
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
},
HOUSE: {
id: "HOUSE",
title: "Kuća",
hasGardenSize: true,
hasAccesRoadType: true,
hasBalconyProp: true,
hasNewBuildingProp: true,
hasElevatorProp: false,
hasNumberOfRoom: true,
hasNumberOfFloors: true,
hasFloorProp: false,
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
},
OFFICE: {
id: "OFFICE",
title: "Kancelarija",
hasGardenSize: false,
hasAccesRoadType: true,
hasBalconyProp: false,
hasNewBuildingProp: true,
hasElevatorProp: true,
hasNumberOfRoom: true,
hasNumberOfFloors: false,
hasFloorProp: true,
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
},
LAND: {
id: "LAND",
title: "Zemljište",
hasGardenSize: false,
hasAccesRoadType: true,
hasBalconyProp: false,
hasNewBuildingProp: false,
hasElevatorProp: false,
hasNumberOfRoom: false,
hasNumberOfFloors: false,
hasFloorProp: false,
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
},
APARTMENT: {
id: "APARTMENT",
title: "Apartman",
hasGardenSize: false,
hasAccesRoadType: true,
hasBalconyProp: true,
hasNewBuildingProp: true,
hasElevatorProp: true,
hasNumberOfRoom: true,
hasNumberOfFloors: false,
hasFloorProp: true,
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
},
GARAGE: {
id: "GARAGE",
title: "Garaža",
hasGardenSize: false,
hasAccesRoadType: true,
hasBalconyProp: false,
hasNewBuildingProp: false,
hasElevatorProp: false,
hasNumberOfRoom: false,
hasNumberOfFloors: false,
hasFloorProp: false,
priceSliderOptionsSale: GARAGE_PRICE_SLIDER_OPTIONS_SALE,
priceSliderOptionsRent: GARAGE_PRICE_SLIDER_OPTIONS_RENT,
sizeSliderOptions: GARAGE_SIZE_SLIDER_OPTIONS
},
COTTAGE: {
id: "COTTAGE",
title: "Vikendica",
hasGardenSize: true,
hasAccesRoadType: true,
hasBalconyProp: true,
hasNewBuildingProp: true,
hasElevatorProp: false,
hasNumberOfRoom: true,
hasNumberOfFloors: true,
hasFloorProp: false,
priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
}
};
const AD_STATUS = {
STATUS_NORMAL: 1,
STATUS_RESERVED: 2,
STATUS_SOLD: 3,
STATUS_DELETED: 4,
STATUS_URGENT: 5,
STATUS_DISCOUNTED: 6,
STATUS_RENTED: 7
};
const AD_AGENCY = {
OLX: "OLX",
RENTAL: "RENTAL",
PROSTOR: "PROSTOR",
AKTIDO: "AKTIDO",
SALJIC: "SALJIC"
};
const CRAWLER_AD_TYPE = {
NONE: 0,
ALL: 1,
ONLY_SELL: 2,
ONLY_RENT: 3,
ONLY_REQUEST: 4
};
const EMAIL_FREQUENCY = {
ASAP: {
id: 1,
stringId: "ASAP",
title: "Odmah"
},
DAILY: {
id: 2,
stringId: "DAILY",
title: "Jednom dnevno"
}
};
const HEATING_TYPE = {
ANY: {
id: "ANY",
title: "Svi"
},
NO_HEATING: {
id: "NO_HEATING",
title: "Nije uvedeno"
},
ELECTRICITY: {
id: "ELECTRICITY",
title: "Struja"
},
GAS: {
id: "GAS",
title: "Plin"
},
WOOD: {
id: "WOOD",
title: "Drva"
},
CENTRAL_CITY: {
id: "CENTRAL_CITY",
title: "Centralno (gradsko)"
},
CENTRAL_BOILER: {
id: "CENTRAL_BOILER",
title: "Centralno (kotlovnica)"
},
CENTRAL_GAS: {
id: "CENTRAL_GAS",
title: "Centralno (plin)"
},
HEAT_PUMP: {
id: "HEAT_PUMP",
title: "Toplotna pumpa"
},
OTHER: {
id: "OTHER",
title: "Drugo"
}
};
const ACCESS_ROAD_TYPE = {
ANY: {
id: "ANY",
title: "Svi"
},
ASPHALT: {
id: "ASPHALT",
title: "Asfalt"
},
CONCRETE: {
id: "CONCRETE",
title: "Beton"
},
MACADAM: {
id: "MACADAM",
title: "Makadam"
},
OTHER: {
id: "OTHER",
title: "Drugo"
}
};
const FURNISHING_TYPE = {
NOT_FURNISHED: {
id: "NOT_FURNISHED",
title: "Nenamješten"
},
HALF_FURNISHED: {
id: "HALF_FURNISHED",
title: "Polunamješten"
},
FURNISHED: {
id: "FURNISHED",
title: "Namješten"
}
};
module.exports = {
AD_TYPE,
AD_CATEGORY,
AD_STATUS,
AD_AGENCY,
CRAWLER_AD_TYPE,
EMAIL_FREQUENCY,
HEATING_TYPE,
ACCESS_ROAD_TYPE,
FURNISHING_TYPE
};

View File

@@ -1,110 +0,0 @@
const { AD_CATEGORY, ACCESS_ROAD_TYPE, HEATING_TYPE } = require("./enums");
const ADVANCED_BOOLEAN_FILTERS = [
{
dbField: "balcony",
title: "Balkon",
categoriesToShow: [
AD_CATEGORY.FLAT,
AD_CATEGORY.HOUSE,
AD_CATEGORY.APARTMENT,
AD_CATEGORY.COTTAGE
]
},
{
dbField: "elevator",
title: "Lift",
categoriesToShow: [
AD_CATEGORY.FLAT,
AD_CATEGORY.APARTMENT,
AD_CATEGORY.OFFICE
]
},
{
dbField: "newBuilding",
title: "Novogradnja",
categoriesToShow: [
AD_CATEGORY.FLAT,
AD_CATEGORY.HOUSE,
AD_CATEGORY.APARTMENT,
AD_CATEGORY.COTTAGE,
AD_CATEGORY.OFFICE
]
}
];
const ADVANCED_SEGMENT_SELECT_FILTERS = [
{
dbField: "accessRoadType",
title: "Pristupni put",
values: Object.keys(ACCESS_ROAD_TYPE).map(key => ACCESS_ROAD_TYPE[key]),
categoriesToShow: [
AD_CATEGORY.FLAT,
AD_CATEGORY.HOUSE,
AD_CATEGORY.APARTMENT,
AD_CATEGORY.COTTAGE,
AD_CATEGORY.OFFICE,
AD_CATEGORY.LAND,
AD_CATEGORY.GARAGE
]
}
// {
// dbField: "heatingType",
// title: "Vrsta grijanja",
// values: Object.keys(HEATING_TYPE).map(key => HEATING_TYPE[key]),
// categoriesToShow: [
// AD_CATEGORY.FLAT,
// AD_CATEGORY.HOUSE,
// AD_CATEGORY.APARTMENT,
// AD_CATEGORY.COTTAGE,
// AD_CATEGORY.OFFICE
// ]
// }
];
const ADVANCED_RANGE_FILTERS = [
{
id: "numberOfFloors",
title: "Broj spratova",
dbFieldMin: "numberOfFloorsMin",
dbFieldMax: "numberOfFloorsMax",
validValueMin: -1,
validValueMax: 50,
categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE]
},
{
id: "floor",
title: "Sprat",
dbFieldMin: "floorMin",
dbFieldMax: "floorMax",
validValueMin: -10,
validValueMax: 50,
categoriesToShow: [
AD_CATEGORY.FLAT,
AD_CATEGORY.APARTMENT,
AD_CATEGORY.OFFICE
]
},
{
id: "numberOfRooms",
title: "Broj soba",
dbFieldMin: "numberOfRoomsMin",
dbFieldMax: "numberOfRoomsMax",
decimalPlaces: 1,
validValueMin: 0,
validValueMax: 200,
categoriesToShow: [
AD_CATEGORY.FLAT,
AD_CATEGORY.HOUSE,
AD_CATEGORY.APARTMENT,
AD_CATEGORY.COTTAGE,
AD_CATEGORY.OFFICE
]
}
];
module.exports = {
ADVANCED_BOOLEAN_FILTERS,
ADVANCED_SEGMENT_SELECT_FILTERS,
ADVANCED_RANGE_FILTERS
};

View File

@@ -1,46 +0,0 @@
"use strict";
require("dotenv").config({ path: __dirname + "/./../../.env" });
const APP_PORT = process.env.PORT || 5000;
const APP_BASE_URL = process.env.APP_BASE_URL || "http://localhost";
const APP_URL =
process.env.NODE_ENV && process.env.NODE_ENV === "production"
? process.env.APP_URL || "http://market-alarm"
: process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
const DEFAULT_TIMEZONE = "Europe/Sarajevo";
const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
const AWS_EMAIL_CONFIG = {
REGION: process.env.AWS_REGION || "",
CREDENTIALS: {
ACCESS_KEY_ID: process.env.AWS_KEY_ID || "",
SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || ""
},
SOURCE_EMAIL: process.env.SOURCE_EMAIL || ""
};
const MAX_REAL_ESTATES_IN_EMAIL =
parseInt(process.env.MAX_REAL_ESTATES_IN_EMAIL) || 10;
const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
parseInt(process.env.MAX_REAL_ESTATES_IN_FIRST_EMAIL) || 5;
const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
const API_MAP_KEY = process.env.API_MAP_KEY || "";
module.exports = {
APP_PORT,
APP_URL,
DEFAULT_TIMEZONE,
CRAWLER_INTERVAL,
STOP_CRAWLER,
AWS_EMAIL_CONFIG,
MAX_REAL_ESTATES_IN_EMAIL,
MAX_REAL_ESTATES_IN_FIRST_EMAIL,
PRINT_CRAWLER_DEBUG,
API_MAP_KEY
};

View File

@@ -1,15 +0,0 @@
{
"development": {
"username": "docker",
"password": "docker",
"database": "marketalerts",
"port": "5432",
"dialect": "postgres"
},
"test": {
"use_env_variable": "DATABASE_URL"
},
"production": {
"use_env_variable": "DATABASE_URL"
}
}

View File

@@ -1,8 +0,0 @@
const getGoAgain = async (req, res) => {
const title = "Uspjeh!";
res.render("goAgain", { title });
};
module.exports = {
getGoAgain
};

View File

@@ -1,85 +0,0 @@
const { currentSearchRequest } = require("../helpers/url");
const getLocation = async (req, res) => {
const title = "Odaberite lokaciju";
const nextStep = req.query.nextStep || "/";
//Check if location data already exists (active request)
//If it does then get location is called through edit field query
//and map should show already selected location not initial map
let selectedLatLngBounds = {};
let boundsSelected = false;
const searchRequest = await currentSearchRequest(req);
if (!searchRequest || !searchRequest.dataValues) {
res.render("notFound", { title: " " });
return;
}
const selectedArea = searchRequest.areaToSearch;
const sw = selectedArea.coordinates[0][3];
const ne = selectedArea.coordinates[0][1];
if (sw[0] && ne[0]) {
selectedLatLngBounds = {
swLat: sw[1],
swLng: sw[0],
neLat: ne[1],
neLng: ne[0]
};
boundsSelected = true;
}
res.render("location", {
nextStep,
title,
boundsSelected,
selectedLatLngBounds
});
};
const postLocation = async (req, res) => {
let searchRequest = await currentSearchRequest(req);
if (!searchRequest || !searchRequest.dataValues) {
res.render("notFound", { title: " " });
return;
}
const northWest = [req.body.west, req.body.north];
const northEast = [req.body.east, req.body.north];
const southEast = [req.body.east, req.body.south];
const southWest = [req.body.west, req.body.south];
const locationInputValue =
req.body.locationInput && req.body.locationInput.length > 0
? req.body.locationInput
: null;
searchRequest.areaToSearch = {
type: "Polygon",
coordinates: [[northWest, northEast, southEast, southWest, northWest]],
crs: { type: "name", properties: { name: "EPSG:4326" } }
};
let locationInputData;
if (req.body.locationInputData) {
try {
locationInputData = JSON.parse(req.body.locationInputData);
} catch (e) {
locationInputData = null;
}
}
await searchRequest.save();
const nextStepPage = req.query.nextStep || "filteri";
const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
res.redirect(nextStepUrl);
};
module.exports = {
getLocation,
postLocation
};

View File

@@ -1,199 +0,0 @@
const { currentSearchRequest } = require("../helpers/url");
const { isValidEmail } = require("../helpers/email");
const {
notifyForNewSearchRequest
} = require("../services/notificationService");
const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
const getQueryReviewTableData = searchRequest => {
const {
id,
adType,
realEstateType,
sizeMin,
sizeMax,
gardenSizeMin,
gardenSizeMax,
priceMin,
priceMax
} = searchRequest.dataValues;
const realEstateTypeObject = AD_CATEGORY[realEstateType];
const enableGardenSizeEdit = realEstateTypeObject
? realEstateTypeObject.hasGardenSize
: false;
let adTypeTitle = "";
switch (adType) {
case AD_TYPE.AD_TYPE_SALE.stringId:
adTypeTitle = AD_TYPE.AD_TYPE_SALE.title;
break;
case AD_TYPE.AD_TYPE_RENT.stringId:
adTypeTitle = AD_TYPE.AD_TYPE_RENT.title;
break;
default:
adTypeTitle = "-";
break;
}
const realEstateTypeTitle = realEstateTypeObject
? `[${adTypeTitle}] ${realEstateTypeObject.title}`
: "-";
const locationTitle = "Promjenite lokaciju";
const sizeTitle = `${sizeMin} - ${sizeMax} m2`;
const gardenSizeTitle = enableGardenSizeEdit
? `${gardenSizeMin} - ${gardenSizeMax} m2`
: "-";
const priceTitle = `${priceMin} - ${priceMax} KM`;
return [
{
id: "realEstateType",
title: realEstateTypeTitle,
url: `/vrstanekretnine/${id}?nextStep=filteri`
},
{
id: "location",
title: locationTitle,
url: `/lokacija/${id}?nextStep=pregled`
},
{
id: "size",
title: sizeTitle,
url: `/filteri/${id}?nextStep=pregled`
},
{
id: "gardenSize",
title: gardenSizeTitle,
url: enableGardenSizeEdit ? `/filteri/${id}?nextStep=pregled` : ""
},
{
id: "price",
title: priceTitle,
url: `/filteri/${id}?nextStep=pregled`
}
].filter(data => data.title != "-");
};
const getQueryReview = async (req, res) => {
const searchRequest = await currentSearchRequest(req);
if (!searchRequest || !searchRequest.dataValues) {
res.render("notFound", { title: " " });
return;
}
const title = "Da li je ovo to što ste tražili ?";
const nextStep = req.query.nextStep;
const error = req.query.error;
const queryReviewTableData = getQueryReviewTableData(searchRequest);
const email = searchRequest.email;
let selectedEmailFrequency;
switch (searchRequest.emailFrequency) {
case EMAIL_FREQUENCY.ASAP.stringId:
selectedEmailFrequency = EMAIL_FREQUENCY.ASAP.id;
break;
case EMAIL_FREQUENCY.DAILY.stringId:
selectedEmailFrequency = EMAIL_FREQUENCY.DAILY.id;
break;
}
res.render("queryReview", {
nextStep,
queryReviewTableData,
title,
email,
selectedEmailFrequency,
error,
EMAIL_FREQUENCY
});
};
const postQueryReview = async (req, res) => {
const searchRequest = await currentSearchRequest(req);
if (!searchRequest || !searchRequest.dataValues) {
return null;
}
const nextStep = req.query.nextStep || "/ponovo";
const emailFrequency =
parseInt(req.body.emailFrequency) || EMAIL_FREQUENCY.ASAP.id;
const emailInput = req.body.email;
const emailConfirmInput = req.body.confirmEmail;
const title = "Da li je ovo to što ste tražili ?";
const queryReviewTableData = getQueryReviewTableData(searchRequest);
let emailFrequencyStringId = EMAIL_FREQUENCY.ASAP.stringId;
if (emailFrequency === EMAIL_FREQUENCY.DAILY.id) {
emailFrequencyStringId = EMAIL_FREQUENCY.DAILY.stringId;
}
searchRequest.emailFrequency = emailFrequencyStringId;
if (emailInput !== emailConfirmInput) {
const error = "Greška ! Unešeni emailovi nisu isti";
res.render("queryReview", {
error,
title,
queryReviewTableData,
email: ""
});
return;
}
if (!isValidEmail(emailInput)) {
const error = "Greška ! Unesite validan email";
res.render("queryReview", {
error,
title,
queryReviewTableData,
email: ""
});
return;
}
searchRequest.email = emailInput;
searchRequest.subscribed = true;
try {
await searchRequest.save();
} catch (e) {
console.log("[ERROR] Failed to save search request !", e);
console.log("Search request : ", searchRequest);
const error =
"Greška ! Nismo uspjeli kreirati zahtjev za Vašu pretragu. Molimo pokuštajte ponovo";
res.render("queryReview", {
error,
title,
queryReviewTableData,
email: ""
});
return;
}
try {
await notifyForNewSearchRequest(searchRequest);
} catch (e) {
console.log("[ERROR] Failed to send initial welcome email", e);
console.log("Search request : ", searchRequest);
const error =
"Greška ! Nismo uspjeli poslati email na Vašu adresu, pokušajte sa drugom email adresom";
res.render("queryReview", {
error,
title,
queryReviewTableData,
email: ""
});
return;
}
res.redirect(nextStep);
};
module.exports = {
getQueryReview,
postQueryReview
};

View File

@@ -1,243 +0,0 @@
const { currentSearchRequest } = require("../helpers/url");
const { AD_CATEGORY, AD_TYPE, ACCESS_ROAD_TYPE } = require("../common/enums");
const {
ADVANCED_BOOLEAN_FILTERS,
ADVANCED_SEGMENT_SELECT_FILTERS,
ADVANCED_RANGE_FILTERS
} = require("../common/filterEnums");
const getFilters = async (req, res) => {
const searchRequest = await currentSearchRequest(req);
if (!searchRequest || !searchRequest.dataValues) {
res.render("notFound", { title: " " });
return;
}
const title = "Filteri za pretraživanje";
const {
adType,
realEstateType,
priceMin,
priceMax,
sizeMin,
sizeMax,
gardenSizeMin,
gardenSizeMax,
numberOfRoomsMin,
numberOfRoomsMax,
numberOfFloorsMin,
numberOfFloorsMax,
floorMin,
floorMax,
includeIncompleteAds,
balcony,
elevator,
newBuilding,
accessRoadType
} = searchRequest;
const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
const {
hasGardenSize,
priceSliderOptionsSale,
priceSliderOptionsRent,
sizeSliderOptions,
gardenSizeSliderOptions
} = category;
let priceSliderOptions;
if (adType === AD_TYPE.AD_TYPE_SALE.stringId) {
priceSliderOptions = Object.assign({}, priceSliderOptionsSale);
} else if (adType === AD_TYPE.AD_TYPE_RENT.stringId) {
priceSliderOptions = Object.assign({}, priceSliderOptionsRent);
} else {
res.render("notFound", { title: " " });
return;
}
// TODO: Maybe this is slow, pay attention to this
const filterFilters = filterObject => {
const filterCategories = filterObject.categoriesToShow;
return filterCategories.indexOf(category) !== -1;
};
const advancedBooleanFilterObjects = ADVANCED_BOOLEAN_FILTERS.filter(
filterFilters
);
const advancedSegmentSelectFilterObjects = ADVANCED_SEGMENT_SELECT_FILTERS.filter(
filterFilters
);
const advancedRangeFilterObjects = ADVANCED_RANGE_FILTERS.filter(
filterFilters
);
const advancedBooleanFilterValues = {
includeIncompleteAds,
balcony,
elevator,
newBuilding
};
const advancedSegmentSelectFilterValues = {
accessRoadType
};
const advancedRangeFilterValues = {
numberOfFloorsMin,
numberOfFloorsMax,
numberOfRoomsMin,
numberOfRoomsMax,
floorMin,
floorMax
};
if (priceMin || priceMax) {
priceSliderOptions.start = [priceMin, priceMax];
}
if (sizeMin || sizeMax) {
sizeSliderOptions.start = [sizeMin, sizeMax];
}
if (gardenSizeSliderOptions && (gardenSizeMin || gardenSizeMax)) {
gardenSizeSliderOptions.start = [gardenSizeMin, gardenSizeMax];
}
res.render("realEstateFilters", {
title,
hasGardenSize,
priceSliderOptions: JSON.stringify(priceSliderOptions),
sizeSliderOptions: JSON.stringify(sizeSliderOptions),
gardenSizeSliderOptions: JSON.stringify(gardenSizeSliderOptions),
advancedBooleanFilterObjects,
advancedBooleanFilterValues,
advancedSegmentSelectFilterObjects,
advancedSegmentSelectFilterValues,
advancedRangeFilterObjects,
advancedRangeFilterValues,
includeIncompleteAds
});
};
const postFilters = async (req, res) => {
const searchRequest = await currentSearchRequest(req);
if (!searchRequest || !searchRequest.dataValues) {
res.render("notFound", { title: " " });
return;
}
const nextStepPage = req.query.nextStep || "pregled";
const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
const priceMin = parseInt(req.body.priceMin) || 0;
const priceMax = parseInt(req.body.priceMax) || 0;
const sizeMin = parseInt(req.body.sizeMin) || 0;
const sizeMax = parseInt(req.body.sizeMax) || 0;
const advancedRangeFilters = {};
ADVANCED_RANGE_FILTERS.forEach(filter => {
let parsingFunction = parseInt;
if (filter.decimalPlaces) {
parsingFunction = parseFloat;
}
advancedRangeFilters[filter.dbFieldMin] = parsingFunction(
req.body[filter.dbFieldMin]
);
advancedRangeFilters[filter.dbFieldMax] = parsingFunction(
req.body[filter.dbFieldMax]
);
advancedRangeFilters[filter.dbFieldMin] = isNaN(
advancedRangeFilters[filter.dbFieldMin]
)
? null
: advancedRangeFilters[filter.dbFieldMin];
advancedRangeFilters[filter.dbFieldMax] = isNaN(
advancedRangeFilters[filter.dbFieldMax]
)
? null
: advancedRangeFilters[filter.dbFieldMax];
try {
if (filter.decimalPlaces) {
advancedRangeFilters[filter.dbFieldMin] = advancedRangeFilters[
filter.dbFieldMin
].toFixed(filter.decimalPlaces);
advancedRangeFilters[filter.dbFieldMax] = advancedRangeFilters[
filter.dbFieldMax
].toFixed(filter.decimalPlaces);
}
} catch (e) {
advancedRangeFilters[filter.dbFieldMin] = null;
advancedRangeFilters[filter.dbFieldMax] = null;
}
if (
advancedRangeFilters[filter.dbFieldMin] < filter.validValueMin ||
advancedRangeFilters[filter.dbFieldMin] > filter.validValueMax
) {
advancedRangeFilters[filter.dbFieldMin] = filter.validValueMin;
}
if (
advancedRangeFilters[filter.dbFieldMax] < filter.validValueMin ||
advancedRangeFilters[filter.dbFieldMax] > filter.validValueMax
) {
advancedRangeFilters[filter.dbFieldMax] = filter.validValueMax;
}
});
const includeIncompleteAds = req.body.includeIncompleteAds === "on";
const balcony = req.body.balcony === "on";
const elevator = req.body.elevator === "on";
const newBuilding = req.body.newBuilding === "on";
const accessRoadType = req.body.accessRoadType;
if (!ACCESS_ROAD_TYPE[accessRoadType]) {
res.render("notFound", { title: " Greška !" });
return;
}
//TODO: Filter validation
searchRequest.priceMin = priceMin;
searchRequest.priceMax = priceMax;
searchRequest.sizeMin = sizeMin;
searchRequest.sizeMax = sizeMax;
for (const filter of Object.keys(advancedRangeFilters)) {
searchRequest[filter] = advancedRangeFilters[filter];
}
searchRequest.balcony = balcony;
searchRequest.elevator = elevator;
searchRequest.newBuilding = newBuilding;
searchRequest.includeIncompleteAds = includeIncompleteAds;
searchRequest.accessRoadType = accessRoadType;
if (
req.body.gardenSizeMin !== undefined &&
req.body.gardenSizeMax !== undefined
) {
const gardenSizeMin = parseInt(req.body.gardenSizeMin);
const gardenSizeMax = parseInt(req.body.gardenSizeMax);
//TODO: Filter validation
searchRequest.gardenSizeMin = gardenSizeMin;
searchRequest.gardenSizeMax = gardenSizeMax;
}
await searchRequest.save();
res.redirect(nextStepUrl);
};
module.exports = {
getFilters,
postFilters
};

View File

@@ -1,81 +0,0 @@
const { currentSearchRequest } = require("../helpers/url");
const { createSearchRequest } = require("../helpers/db/searchRequest");
const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
const getRealEstateTypes = async (req, res) => {
const searchRequest = await currentSearchRequest(req);
const title = "Koju nekretninu tražite?";
let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
if (
searchRequest &&
searchRequest.adType &&
searchRequest.adType === AD_TYPE.AD_TYPE_RENT.stringId
) {
selectedAdType = AD_TYPE.AD_TYPE_RENT.id;
}
const realEstateTypes = Object.keys(AD_CATEGORY)
.map(category => AD_CATEGORY[category])
.filter(category => category.title);
res.render("realEstateType", {
selectedAdType,
realEstateTypes,
title,
AD_TYPE
});
};
const postRealEstateTypes = async (req, res) => {
const searchRequest = await currentSearchRequest(req);
const adType = parseInt(req.body.adType);
const adTypeStringIds = {
[AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
[AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
};
const adTypeStringId =
adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
const validRealEstateTypes = Object.keys(AD_CATEGORY).filter(
category => !!AD_CATEGORY[category].title
);
const selectedRealEstateType = req.body.realEstateType || null;
if (validRealEstateTypes.indexOf(selectedRealEstateType) === -1) {
res.render("notFound", { title: " " });
return;
}
const nextStepPage = req.query.nextStep || "lokacija";
let nextStepUrl = "";
if (searchRequest && searchRequest.id) {
nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
searchRequest.adType = adTypeStringId;
searchRequest.realEstateType = selectedRealEstateType;
await searchRequest.save();
} else {
try {
const newSearchRequest = await createSearchRequest({
adType: adTypeStringId,
realEstateType: selectedRealEstateType
});
nextStepUrl = `/${nextStepPage}/${newSearchRequest.id}`;
} catch (error) {
console.log(error);
nextStepUrl = `/`;
}
}
res.redirect(nextStepUrl);
};
module.exports = {
getRealEstateTypes,
postRealEstateTypes
};

View File

@@ -1,16 +0,0 @@
"use strict";
const {
findRealEstatesForSearchRequest
} = require("../helpers/db/searchRequestMatch");
const getRealEstates = async (req, res) => {
const searchRequestId = req.params["searchRequestId"] || "";
const realEstates = await findRealEstatesForSearchRequest(searchRequestId);
const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage";
res.render("realEstates", { realEstates, title });
};
module.exports = {
getRealEstates
};

View File

@@ -1,33 +0,0 @@
const { getRealEstateById } = require("../helpers/db/realEstate");
const getRedirect = async (req, res) => {
const id = req.params.id || null;
let error = false;
let redirectUrl = undefined;
if (!id) {
error = true;
} else {
try {
const realEstate = await getRealEstateById(id);
if (!realEstate) {
error = true;
} else {
redirectUrl = realEstate.url;
}
} catch (e) {
error = true;
}
}
if (error) {
const title = "";
res.render("notFound", { title });
} else {
const title = "Preusmjeravanje";
res.render("redirect", { title, redirectUrl });
}
};
module.exports = {
getRedirect
};

View File

@@ -1,22 +0,0 @@
const { currentSearchRequest } = require("../helpers/url");
const getUnsubscribe = async (req, res) => {
const title = "Uspješno ste se odjavili";
const searchRequest = await currentSearchRequest(req);
if (!searchRequest || !searchRequest.dataValues) {
res.render("notFound", { title: " " });
return;
}
searchRequest.subscribed = false;
searchRequest.deletedEmail = searchRequest.email;
searchRequest.email = "";
await searchRequest.save();
res.render("unsubscribe", { nextStep: "/vrstanekretnine", title });
};
module.exports = {
getUnsubscribe
};

View File

@@ -1,42 +0,0 @@
const { createSearchRequest } = require("../helpers/db/searchRequest");
const { AD_TYPE, AD_CATEGORY } = require("../common/enums");
const getWelcome = (req, res) => {
res.render("welcome", {
title: false,
AD_TYPE
});
};
const postWelcome = async (req, res) => {
const adType = parseInt(req.body.adType);
const adTypeStringIds = {
[AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
[AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
};
const adTypeStringId =
adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
let nextStepUrl = "";
try {
const newSearchRequest = await createSearchRequest({
adType: adTypeStringId,
realEstateType: AD_CATEGORY.FLAT.id
});
nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
} catch (error) {
console.log(error);
nextStepUrl = `/`;
}
res.redirect(nextStepUrl);
};
module.exports = {
getWelcome,
postWelcome
};

View File

@@ -1,92 +0,0 @@
"use strict";
/*
Entry point for crawling functionality
All communication between crawlers and savers is here
All environment specific configuration is read here and
passed to the crawlers and savers.
*/
const OlxCrawler = require("./specificCrawlers/olx");
const RentalCrawler = require("./specificCrawlers/rental");
const ProstorCrawler = require("./specificCrawlers/prostor");
const AktidoCrawler = require("./specificCrawlers/aktido");
const SaljicCrawler = require("./specificCrawlers/saljic");
const {
OLX_CONFIG,
RENTAL_CONFIG,
PROSTOR_CONFIG,
AKTIDO_CONFIG,
SALJIC_CONFIG
} = require("./crawlerConfig");
const PostgresSaver = require("./savers/postgres");
async function crawlAll() {
const postgresSaver = new PostgresSaver();
const crawlers = [
new OlxCrawler(
[postgresSaver],
OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
OLX_CONFIG.OLX_MAX_PAGES,
OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
OLX_CONFIG.OLX_IGNORED_USERNAMES,
OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
),
new RentalCrawler(
[postgresSaver],
RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
RENTAL_CONFIG.RENTAL_MAX_PAGES,
RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
),
new ProstorCrawler(
[postgresSaver],
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
),
new AktidoCrawler(
[postgresSaver],
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
),
new SaljicCrawler(
[postgresSaver],
SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
SALJIC_CONFIG.SALJIC_MAX_PAGES,
SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
)
];
const newRealEstates = [];
for (const crawler of crawlers) {
try {
const newRealEstatesFromSingleCrawler = await crawler.crawl();
if (Array.isArray(newRealEstatesFromSingleCrawler)) {
newRealEstates.push(...newRealEstatesFromSingleCrawler);
}
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
}
}
return newRealEstates;
}
module.exports = {
crawlAll
};

View File

@@ -1,16 +0,0 @@
"use strict";
require("dotenv").config({ path: __dirname + "/./../../.env" });
const OLX_CONFIG = require("./specificConfigs/olx");
const RENTAL_CONFIG = require("./specificConfigs/rental");
const PROSTOR_CONFIG = require("./specificConfigs/prostor");
const AKTIDO_CONFIG = require("./specificConfigs/aktido");
const SALJIC_CONFIG = require("./specificConfigs/saljic");
module.exports = {
OLX_CONFIG,
RENTAL_CONFIG,
PROSTOR_CONFIG,
AKTIDO_CONFIG,
SALJIC_CONFIG
};

View File

@@ -1,5 +0,0 @@
const { crawlAll } = require("./crawl");
(async () => {
await crawlAll();
})();

View File

@@ -1,47 +0,0 @@
const moment = require("moment");
const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
class PostgresSaver {
connect() {
//TODO: It seems we never worry about open/close connection with Sequelize ?
//TODO: Check if postgres is ready
return true;
}
async save(results) {
const savedRecords = await bulkUpsertRealEstates(results);
if (Array.isArray(savedRecords)) {
const newRealEstates = [];
const existingRealEstates = [];
for (const savedRecord of savedRecords) {
const { createdAt, updatedAt } = savedRecord;
const createdAtMoment = moment.utc(createdAt);
const updatedAtMoment = moment.utc(updatedAt);
if (createdAtMoment.isSame(updatedAtMoment, "second")) {
newRealEstates.push(savedRecord);
} else {
existingRealEstates.push(savedRecord);
}
}
return {
newRecords: newRealEstates,
existingRecords: existingRealEstates
};
} else {
throw { message: "[POSTGRES] Failed to save records" };
}
}
close() {
//TODO: It seems we never worry about open/close connection with Sequelize ?
return true;
}
}
module.exports = PostgresSaver;

View File

@@ -1,34 +0,0 @@
"use strict";
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
const aktidoCrawlerAdType =
process.env.AKTIDO_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.AKTIDO_CRAWLER_AD_TYPE]
: null;
const aktidoParsedCrawlerAdCategories =
process.env.AKTIDO_CRAWLER_AD_CATEGORIES !== undefined
? process.env.AKTIDO_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const aktidoIgnoredUsernames = [];
const transformedAktidoCrawlerAdCategories = aktidoParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
module.exports = {
AKTIDO_MAX_PAGES: parseInt(process.env.AKTIDO_MAX_PAGES) || 500,
AKTIDO_MAX_RESULTS_PER_PAGE:
parseInt(process.env.AKTIDO_MAX_RESULTS_PER_PAGE) || 50,
AKTIDO_CRAWLER_AD_TYPE: aktidoCrawlerAdType || CRAWLER_AD_TYPE.NONE,
AKTIDO_CRAWLER_AD_CATEGORIES: transformedAktidoCrawlerAdCategories,
AKTIDO_IGNORED_USERNAMES: aktidoIgnoredUsernames || [],
AKTIDO_DELAY_BETWEEN_PAGES:
parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000,
AKTIDO_FORCE_CRAWL: !!parseInt(process.env.AKTIDO_FORCE_CRAWL)
};

View File

@@ -1,39 +0,0 @@
"use strict";
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
const olxCrawlerAdType =
process.env.OLX_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
: null;
const olxParsedCrawlerAdCategories =
process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const olxIgnoredUsernames =
process.env.OLX_IGNORED_USERNAMES !== undefined
? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
username.trim()
)
: [];
const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
module.exports = {
OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
OLX_MAX_RESULTS_PER_PAGE:
parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
OLX_DELAY_BETWEEN_PAGES:
parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000,
OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL)
};

View File

@@ -1,34 +0,0 @@
"use strict";
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
const prostorCrawlerAdType =
process.env.PROSTOR_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.PROSTOR_CRAWLER_AD_TYPE]
: null;
const prostorParsedCrawlerAdCategories =
process.env.PROSTOR_CRAWLER_AD_CATEGORIES !== undefined
? process.env.PROSTOR_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const prostorIgnoredUsernames = [];
const transformedProstorCrawlerAdCategories = prostorParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
module.exports = {
PROSTOR_MAX_PAGES: parseInt(process.env.PROSTOR_MAX_PAGES) || 100,
PROSTOR_MAX_RESULTS_PER_PAGE:
parseInt(process.env.PROSTOR_MAX_RESULTS_PER_PAGE) || 5000,
PROSTOR_CRAWLER_AD_TYPE: prostorCrawlerAdType || CRAWLER_AD_TYPE.NONE,
PROSTOR_CRAWLER_AD_CATEGORIES: transformedProstorCrawlerAdCategories,
PROSTOR_IGNORED_USERNAMES: prostorIgnoredUsernames || [],
PROSTOR_DELAY_BETWEEN_PAGES:
parseInt(process.env.PROSTOR_DELAY_BETWEEN_PAGES) || 1000,
PROSTOR_FORCE_CRAWL: !!parseInt(process.env.PROSTOR_FORCE_CRAWL)
};

View File

@@ -1,34 +0,0 @@
"use strict";
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
const rentalCrawlerAdType =
process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
: null;
const rentalParsedCrawlerAdCategories =
process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const rentalIgnoredUsernames = [];
const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
module.exports = {
RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
RENTAL_MAX_RESULTS_PER_PAGE:
parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
RENTAL_DELAY_BETWEEN_PAGES:
parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000,
RENTAL_FORCE_CRAWL: !!parseInt(process.env.RENTAL_FORCE_CRAWL)
};

View File

@@ -1,34 +0,0 @@
"use strict";
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
const saljicCrawlerAdType =
process.env.SALJIC_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.SALJIC_CRAWLER_AD_TYPE]
: null;
const saljicParsedCrawlerAdCategories =
process.env.SALJIC_CRAWLER_AD_CATEGORIES !== undefined
? process.env.SALJIC_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const saljicIgnoredUsernames = [];
const transformedSaljicCrawlerAdCategories = saljicParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
module.exports = {
SALJIC_MAX_PAGES: parseInt(process.env.SALJIC_MAX_PAGES) || 100,
SALJIC_MAX_RESULTS_PER_PAGE:
parseInt(process.env.SALJIC_MAX_RESULTS_PER_PAGE) || 5000,
SALJIC_CRAWLER_AD_TYPE: saljicCrawlerAdType || CRAWLER_AD_TYPE.NONE,
SALJIC_CRAWLER_AD_CATEGORIES: transformedSaljicCrawlerAdCategories,
SALJIC_IGNORED_USERNAMES: saljicIgnoredUsernames || [],
SALJIC_DELAY_BETWEEN_PAGES:
parseInt(process.env.SALJIC_DELAY_BETWEEN_PAGES) || 1000,
SALJIC_FORCE_CRAWL: !!parseInt(process.env.SALJIC_FORCE_CRAWL)
};

View File

@@ -1,846 +0,0 @@
"use strict";
const fetch = require("node-fetch");
const cheerio = require("cheerio");
const Promise = require("bluebird");
const moment = require("moment-timezone");
const htmlToText = require("html-to-text");
const {
AD_TYPE,
AD_CATEGORY,
AD_AGENCY,
AD_STATUS,
CRAWLER_AD_TYPE,
HEATING_TYPE,
ACCESS_ROAD_TYPE,
FURNISHING_TYPE
} = require("../../common/enums");
const {
DEFAULT_TIMEZONE,
PRINT_CRAWLER_DEBUG
} = require("../../config/appConfig");
const AKTIDO_ENUMS = {
AKTIDO_AD_TYPE: {
[CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
[CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
[CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
},
AKTIDO_AD_CATEGORY: {
[AD_CATEGORY.ALL.id]: "",
[AD_CATEGORY.FLAT.id]: "/tip-2",
[AD_CATEGORY.HOUSE.id]: "/tip-1",
[AD_CATEGORY.LAND.id]: "/tip-5",
[AD_CATEGORY.OFFICE.id]: "/tip-4",
[AD_CATEGORY.APARTMENT.id]: "/tip-3",
[AD_CATEGORY.GARAGE.id]: "/tip-6"
//[AD_CATEGORY.COTTAGE.id]: ""
},
AKTIDO_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
AKTIDO_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
};
const { AKTIDO_FORCE_CRAWL } = require("../specificConfigs/aktido");
class AktidoCrawler {
constructor(
savers = [],
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
maxPages = 1000,
maxResultsPerPage = 100,
ignoredUsernames = [],
delayBetweenPages = 1000
) {
this.savers = savers;
this.baseUrl = "https://www.aktido.ba/pretraga/sortiraj-date_DESC";
this.crawlerAdTypes = crawlerAdTypes;
this.crawlerAdCategories = crawlerAdCategories;
this.maxPages = maxPages;
this.maxResultsPerPage = maxResultsPerPage;
this.delayBetweenPages = delayBetweenPages;
}
async crawl() {
const crawlAdCategories = this.crawlerAdCategories;
const newRealEstates = [];
if (crawlAdCategories) {
const indexGenerators = [];
for (const adCategory of crawlAdCategories) {
indexGenerators.push(this.categoryIndexer(adCategory));
}
let done = false;
while (!done) {
const categoryIndexerPromises = [];
const generatorsToRemove = [];
for (const indexGenerator of indexGenerators) {
categoryIndexerPromises.push(indexGenerator.next());
generatorsToRemove.push(false);
}
const singlePageResults = await Promise.all(categoryIndexerPromises);
const entries = singlePageResults.entries();
for (const [index, { value: singlePageResult }] of entries) {
if (singlePageResult) {
const saveResults = await this.saveCrawledResults(singlePageResult);
const { newRecords } = saveResults;
newRealEstates.push(...newRecords);
if (
Array.isArray(newRecords) &&
newRecords.length === 0 &&
!AKTIDO_FORCE_CRAWL
) {
generatorsToRemove[index] = true;
}
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;
// console.log("Generator ", index + 1, "has no more pages");
}
}
// console.log("Generators state : ", generatorsToRemove);
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
if (generatorsToRemove[i]) {
// console.log("\tRemove generator ", i + 1);
indexGenerators.splice(i, 1);
}
}
if (indexGenerators.length === 0) {
done = true;
}
await this.sleep(this.delayBetweenPages);
}
}
return newRealEstates;
}
async *categoryIndexer(adCategory) {
let pageToIndex = 1;
const urlAdTypePart = AKTIDO_ENUMS.AKTIDO_AD_TYPE[this.crawlerAdTypes];
const urlCategoryPart = AKTIDO_ENUMS.AKTIDO_AD_CATEGORY[adCategory];
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
while (true) {
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
const singlePageResults = await this.indexSinglePage(
urlPageToCrawl,
this.maxResultsPerPage
);
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
yield singlePageResults;
} else {
return undefined;
}
++pageToIndex;
if (pageToIndex === this.maxPages) {
return undefined;
}
}
} else {
return undefined;
}
}
async indexSinglePage(url, maxResultsPerPage) {
if (PRINT_CRAWLER_DEBUG) {
console.log("[AKTIDO] Index page : ", url);
}
try {
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
let hrefs = [];
$(
"body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
)
.find(".moreInfo")
.each((i, elem) => {
const href = $(elem)
.find("a")
.first()
.attr("href");
if (href) {
hrefs.push(href);
}
});
let actualNoOfResults =
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefs[i]));
}
const scrapedData = await Promise.all(asyncScraping);
const filteredScrapedData = scrapedData.filter(adData => !!adData);
return filteredScrapedData;
} catch (e) {
console.error("[AKTIDO] Exception caught:" + e);
return [];
}
}
async scrapeAd(url) {
// console.log("[AKTIDO] Scraping : ", url);
try {
const adPageSource = await fetch(url);
const body = await adPageSource.text();
const $ = cheerio.load(body);
const mapElementParent = $(".box-map").parent();
const scriptElement = $("script", mapElementParent);
if (
scriptElement[0] &&
scriptElement[0].children &&
scriptElement[0].children[0] &&
scriptElement[0].children[0].data
) {
let extractedData;
try {
//data string starts with : var json_map_data = [{"r ...
//so we remove first 20 characters
const jsonData = scriptElement[0].children[0].data.substring(20);
const parsedJsonData = JSON.parse(jsonData);
extractedData = parsedJsonData[0];
} catch (e) {
throw { message: "Can't find ad data JSON" };
}
let adStatus = AD_STATUS.STATUS_NORMAL;
const aktidoId = extractedData["re_realEstates_id"];
const adCategory = this.getKiviCategoryIdFromAktidoId(
parseInt(extractedData["re_types_id"])
);
if (!adCategory) {
throw {
message: `Invalid category : ${extractedData["re_types_id"]}`
};
}
const adType = this.getKiviAdTypeFromAktidoActionId(
parseInt(extractedData["re_action_id"])
);
if (!adType) {
throw {
message: `Invalid ad type : ${extractedData["re_action_id"]}`
};
}
const descriptionIds = extractedData["re_descriptions_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(descriptionIds)) {
throw {
message:
'Expected array od descriptions but "re_descriptions_id" not found !'
};
}
const spaceIds = extractedData["re_spaces_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(spaceIds)) {
throw {
message: 'Expected array od spaces but "re_spaces_id" not found !'
};
}
const infrastructureIds = extractedData["re_infrastructure_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(infrastructureIds)) {
throw {
message:
'Expected array od infrastructures but "re_infrastructure_id" not found !'
};
}
const floorNoIds = extractedData["re_floorNO_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(floorNoIds)) {
throw {
message:
'Expected array od infrastructures but "re_floorNO_id" not found !'
};
}
// counting floor enums
// for (let i = 1; i < 10; i++) {
// const floorEnumsTitle = $(
// `body > div.container-fluid > div.container > div:nth-child(2) > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div:nth-child(1) > div > div > div.col-xs-12.col-sm-4.box-details > div.body > p:nth-child(${i}) > span:nth-child(1)`
// )
// .text()
// .trim();
// if (floorEnumsTitle === "Spratnost:") {
// const floorEnumsValue = $(
// `body > div.container-fluid > div.container > div:nth-child(2) > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div:nth-child(1) > div > div > div.col-xs-12.col-sm-4.box-details > div.body > p:nth-child(${i}) > span:nth-child(2)`
// )
// .text()
// .trim()
// .split(",");
//
// console.log("==========");
// floorNoIds.forEach((id, index) => {
// console.log("\t", id, " = ", floorEnumsValue[index]);
// });
// break;
// }
// }
// enumerating infrastructure - relation between id and infrastructure title
// let found = false;
// let infrastructureDescriptions = {};
// for (let i = 1; i < 5; i++) {
// found = false;
// for (let j = 1; j < 10; j++) {
// const infrastructureTitle = $(
// `#b2 > div > div:nth-child(${i}) > div > ul > li:nth-child(${j}) > strong`
// )
// .text()
// .trim();
// if (infrastructureTitle === "Osnovna infrastruktura:") {
// found = true;
//
// const infrastructureValues = $(
// `#b2 > div > div:nth-child(${i}) > div > ul > li:nth-child(${j}) > div`
// )
// .text()
// .trim()
// .split(",");
//
// infrastructureIds.forEach((id, index) => {
// infrastructureDescriptions[id] = infrastructureValues[index];
// });
// }
// }
// if (found) {
// break;
// }
// }
const realEstatePropertiesFromDescriptions = this.getPropertiesFromDescriptions(
descriptionIds
);
const realEstatePropertiesFromSpaces = this.getPropertiesFromSpaces(
spaceIds
);
const realEstatePropertiesFromInfrastructure = this.getPropertiesFromInfrastructure(
infrastructureIds
);
if (extractedData["adm_realEstates_discount"] === "1") {
adStatus = AD_STATUS.STATUS_DISCOUNTED;
}
let numberOfRooms =
parseInt(extractedData["re_realEstates_roomsNO"]) +
parseInt(extractedData["re_realEstates_bedroomNO"]) || null,
numberOfFloors =
parseInt(extractedData["re_realEstates_floorsNO"]) ||
this.getNumberOfFloorsFromFloorId(extractedData["re_floorNO_id"]),
floor =
parseInt(extractedData["re_realEstates_floorNO"]) ||
this.getFloorNumberFromFloorId(extractedData["re_floorNO_id"]),
accessRoadType = realEstatePropertiesFromDescriptions.accessRoadType,
heatingType =
this.getHeatingTypeId(extractedData["re_heating_id"]) || null,
furnishingType = realEstatePropertiesFromDescriptions.furnishingType,
balcony =
realEstatePropertiesFromDescriptions.balcony ||
realEstatePropertiesFromSpaces.balcony,
newBuilding = extractedData["op_realEstates_newBuilding"]
? extractedData["op_realEstates_newBuilding"] === "1"
: null,
elevator = realEstatePropertiesFromDescriptions.elevator,
water =
realEstatePropertiesFromDescriptions.water ||
realEstatePropertiesFromInfrastructure.water,
electricity =
realEstatePropertiesFromDescriptions.electricity ||
realEstatePropertiesFromInfrastructure.electricity,
drainageSystem =
realEstatePropertiesFromInfrastructure.drainageSystem,
registeredInZkBooks =
extractedData["op_realEstates_ownerPermit"] === 1 || null,
recentlyAdapted = null,
parking =
realEstatePropertiesFromDescriptions.parking ||
realEstatePropertiesFromSpaces.parking,
garage = realEstatePropertiesFromSpaces.garage,
gas = realEstatePropertiesFromInfrastructure.gas,
antiTheftDoor = realEstatePropertiesFromDescriptions.antiTheftDoor,
airCondition = realEstatePropertiesFromDescriptions.airCondition,
phoneConnection =
realEstatePropertiesFromInfrastructure.phoneConnection,
cableTV = realEstatePropertiesFromInfrastructure.cableTV,
internet = realEstatePropertiesFromInfrastructure.internet,
basementAttic = realEstatePropertiesFromSpaces.basementAttic,
storeRoom = realEstatePropertiesFromSpaces.storeRoom,
videoSurveillance =
realEstatePropertiesFromDescriptions.videoSurveillance ||
realEstatePropertiesFromInfrastructure.videoSurveillance,
alarm = realEstatePropertiesFromDescriptions.alarm,
suitableForStudents = null,
includingBills =
extractedData["op_realEstates_utilitiesIncluded"] === "1" || null,
animalsAllowed = null,
pool = realEstatePropertiesFromDescriptions.pool,
urbanPlanPermit =
extractedData["op_realEstates_locationPermit"] === "1" ||
realEstatePropertiesFromDescriptions.urbanPlanPermit,
buildingPermit =
extractedData["op_realEstates_buildingPermit"] === "1" || null,
utilityConnection =
realEstatePropertiesFromDescriptions.utilityConnection,
distanceToRiver = null,
numberOfViewsAgency = null;
const title = extractedData["re_realEstates_portalName"];
const extractedPrice = parseFloat(
extractedData["re_realEstates_price"]
);
const price = extractedPrice ? extractedPrice : null;
const area = parseFloat(extractedData["re_realEstates_area"]);
const gardenSize = parseFloat(
extractedData["re_realEstates_fieldArea"]
);
const longDescription = htmlToText.fromString(
extractedData["re_realEstates_description"]
);
const locationLong = extractedData["re_realEstates_longitude"];
const locationLat = extractedData["re_realEstates_latitude"];
const publishedDateMoment = moment.tz(
extractedData["re_realEstates_inserted"],
AKTIDO_ENUMS.AKTIDO_PUBLISHED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!publishedDateMoment.isValid()) {
throw {
message: `Invalid published date : ${
extractedData["re_realEstates_inserted"]
}`
};
}
const renewedDateMoment = moment.tz(
extractedData["re_realEstates_edited"],
AKTIDO_ENUMS.AKTIDO_RENEWED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!renewedDateMoment.isValid()) {
throw {
message: `Invalid renewed date : ${
extractedData["re_realEstates_edited"]
}`
};
}
const data = {
url,
agencyObjectId: aktidoId,
originAgencyName: AD_AGENCY.AKTIDO,
realEstateType: adCategory,
adType,
title,
price,
area,
gardenSize,
shortDescription: "",
longDescription: longDescription,
streetNumber: 0,
streetName: "",
locality: "",
municipality: "",
city: "",
region: "",
entity: "",
country: "",
locationLat,
locationLong,
adStatus,
publishedDate: publishedDateMoment.toISOString(),
renewedDate: renewedDateMoment.toISOString(),
numberOfRooms,
numberOfFloors,
floor,
accessRoadType,
heatingType,
furnishingType,
balcony,
newBuilding,
elevator,
water,
electricity,
drainageSystem,
registeredInZkBooks,
recentlyAdapted,
parking,
garage,
gas,
antiTheftDoor,
airCondition,
phoneConnection,
cableTV,
internet,
basementAttic,
storeRoom,
videoSurveillance,
alarm,
suitableForStudents,
includingBills,
animalsAllowed,
pool,
urbanPlanPermit,
buildingPermit,
utilityConnection,
distanceToRiver,
numberOfViewsAgency
};
return data;
} else {
console.log("[AKTIDO] No JSON data for this ad : ", url);
return null;
}
} catch (e) {
console.error("[AKTIDO] Exception caught: " + e.message, "\r\nURL:", url);
return null;
}
return null;
}
//======= HELPER FUNCTIONS =============
getKiviCategoryIdFromAktidoId(aktidoCategoryId) {
switch (aktidoCategoryId) {
case 1:
return AD_CATEGORY.HOUSE.id;
case 2:
return AD_CATEGORY.FLAT.id;
case 3:
return AD_CATEGORY.APARTMENT.id;
case 4:
return AD_CATEGORY.OFFICE.id;
case 5:
return AD_CATEGORY.LAND.id;
case 6:
return AD_CATEGORY.GARAGE.id;
default:
return undefined;
}
}
getKiviAdTypeFromAktidoActionId(actionId) {
switch (actionId) {
case 1:
return AD_TYPE.AD_TYPE_SALE.stringId;
case 2:
return AD_TYPE.AD_TYPE_RENT.stringId;
default:
return undefined;
}
}
getPropertiesFromDescriptions(descriptionIds) {
const result = {
accessRoadType: null,
furnishingType: null,
balcony: null,
elevator: null,
parking: null,
antiTheftDoor: null,
airCondition: null,
videoSurveillance: null,
alarm: null,
pool: null,
urbanPlanPermit: null,
utilityConnection: null,
water: null,
electricity: null
};
for (const descriptionId of descriptionIds) {
switch (descriptionId) {
case 16:
result.furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
break;
case 17:
result.furnishingType = FURNISHING_TYPE.HALF_FURNISHED.id;
break;
case 1:
case 28:
result.furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case 14:
result.elevator = true;
break;
case 39:
result.electricity = true;
break;
case 40:
result.water = true;
break;
case 41:
case 58:
result.accessRoadType = ACCESS_ROAD_TYPE.ASPHALT.id;
break;
case 26:
result.balcony = true;
break;
case 62:
result.parking = true;
break;
case 3:
result.antiTheftDoor = true;
break;
case 2:
case 21:
result.airCondition = true;
break;
case 4:
result.alarm = true;
break;
case 55:
result.videoSurveillance = true;
break;
case 9:
result.pool = true;
break;
case 60:
result.urbanPlanPermit = true;
break;
case 38:
result.utilityConnection = true;
break;
}
}
return result;
}
getPropertiesFromSpaces(spaceIds) {
const result = {
balcony: null,
parking: null,
garage: null,
basementAttic: null,
storeRoom: null
};
for (const spaceId of spaceIds) {
switch (spaceId) {
case 36:
case 12:
result.parking = true;
break;
case 1:
case 2:
case 3:
result.balcony = true;
break;
case 4:
case 30:
result.garage = true;
break;
case 9:
case 10:
result.storeRoom = true;
break;
case 18:
case 34:
case 37:
case 27:
result.basementAttic = true;
break;
}
}
return result;
}
getHeatingTypeId(heatingRentalId) {
// heatingRentalId can have multiple values, like: "1, 2, 3", parseInt will take first integer value
const heatingId = parseInt(heatingRentalId);
switch (heatingId) {
case 27:
case 16:
return HEATING_TYPE.GAS.id;
case 4:
return HEATING_TYPE.CENTRAL_GAS.id;
case 3:
case 23:
case 6:
case 7:
case 8:
case 9:
case 10:
return HEATING_TYPE.CENTRAL_BOILER.id;
case 2:
case 13:
case 30:
case 17:
case 29:
case 31:
return HEATING_TYPE.ELECTRICITY.id;
case 24:
case 25:
case 12:
return HEATING_TYPE.CENTRAL_CITY.id;
case 26:
case 21:
case 20:
return HEATING_TYPE.WOOD.id;
case 28:
case 19:
return HEATING_TYPE.HEAT_PUMP.id;
case 14:
case 32:
return HEATING_TYPE.OTHER.id;
default:
return null;
}
}
getPropertiesFromInfrastructure(infrastructureIds) {
const result = {
electricity: null,
water: null,
gas: null,
drainageSystem: null,
phoneConnection: null,
internet: null,
videoSurveillance: null,
cableTV: null
};
for (const infrastructureId of infrastructureIds) {
switch (infrastructureId) {
case 1:
result.electricity = true;
break;
case 2:
result.water = true;
break;
case 4:
result.gas = true;
break;
case 5:
result.drainageSystem = true;
break;
case 7:
case 8:
result.phoneConnection = true;
break;
case 10:
result.internet = true;
break;
case 11:
result.cableTV = true;
break;
case 16:
case 17:
result.videoSurveillance = true;
break;
}
}
return result;
}
getFloorNumberFromFloorId(floorsIdText) {
// floorIdText can be array of numbers, separated by comma or number
// just extracting floor number from first element
const floorsId = floorsIdText.split(",");
if (floorsId.length === 0) {
return null;
}
const firstFloorId = parseInt(floorsId[0]);
// 1 pod
// 2 sut
// 3 raz
// 4 pri
// 5 vpri
// 6 prv
// 7 dru
// 8 tre
// 9 čet
// 10 man
// 11
// 12 pot
// 13 vpot
// 14 tav
// 15 pet
const floorNumber = [
-1,
-1,
0,
0,
1,
1,
2,
3,
4,
null,
null,
null,
null,
null,
5
];
return floorNumber[firstFloorId - 1] || null;
}
getNumberOfFloorsFromFloorId(floorsIdText) {
// floorIdText can be array of numbers, separated by comma or number
const floorIds = floorsIdText.split(",");
if (floorIds.length === 0) {
return null;
}
return floorIds.length;
}
async sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async saveCrawledResults(results) {
const savers = this.savers;
// for (const saver of savers) {
// await saver.save(results);
// }
//For now, we use only Postgres saver, so ...
return savers[0].save(results);
//so that we can use some sequelize options and information when data is inserted
}
}
module.exports = AktidoCrawler;

View File

@@ -1,888 +0,0 @@
"use strict";
const fetch = require("node-fetch");
const cheerio = require("cheerio");
const Promise = require("bluebird");
const moment = require("moment-timezone");
const {
AD_TYPE,
AD_CATEGORY,
AD_AGENCY,
AD_STATUS,
CRAWLER_AD_TYPE,
HEATING_TYPE,
FURNISHING_TYPE,
ACCESS_ROAD_TYPE
} = require("../../common/enums");
const {
DEFAULT_TIMEZONE,
PRINT_CRAWLER_DEBUG
} = require("../../config/appConfig");
const OLX_ENUMS = {
OLX_AD_TYPE: {
[CRAWLER_AD_TYPE.ALL]: "",
[CRAWLER_AD_TYPE.ONLY_SELL]: "&vrsta=samoprodaja",
[CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje",
[CRAWLER_AD_TYPE.ONLY_REQUEST]: "&vrsta=samopotraznja"
},
OLX_AD_CATEGORY: {
[AD_CATEGORY.FLAT.id]: "&kategorija=23",
[AD_CATEGORY.HOUSE.id]: "&kategorija=24",
[AD_CATEGORY.LAND.id]: "&kategorija=29",
[AD_CATEGORY.OFFICE.id]: "&kategorija=25",
[AD_CATEGORY.APARTMENT.id]: "&kategorija=27",
[AD_CATEGORY.GARAGE.id]: "&kategorija=30",
[AD_CATEGORY.COTTAGE.id]: "&kategorija=26"
},
MAX_DETAIL_FIELDS: 30,
OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
};
const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
class OlxCrawler {
constructor(
savers = [],
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
maxPages = 1000,
maxResultsPerPage = 100,
ignoredUsernames = [],
delayBetweenPages = 1000
) {
this.savers = savers;
this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
this.crawlerAdTypes = crawlerAdTypes;
this.crawlerAdCategories = crawlerAdCategories;
this.maxPages = maxPages;
this.maxResultsPerPage = maxResultsPerPage;
this.ignoredUsernames = ignoredUsernames;
this.delayBetweenPages = delayBetweenPages;
}
async crawl() {
const crawlAdCategories = this.crawlerAdCategories;
const newRealEstates = [];
if (crawlAdCategories) {
const indexGenerators = [];
for (const adCategory of crawlAdCategories) {
indexGenerators.push(this.categoryIndexer(adCategory));
}
let done = false;
while (!done) {
const categoryIndexerPromises = [];
const generatorsToRemove = [];
for (const indexGenerator of indexGenerators) {
categoryIndexerPromises.push(indexGenerator.next());
generatorsToRemove.push(false);
}
const singlePageResults = await Promise.all(categoryIndexerPromises);
const entries = singlePageResults.entries();
for (const [index, { value: singlePageResult }] of entries) {
if (singlePageResult) {
const saveResults = await this.saveCrawledResults(singlePageResult);
const { newRecords, existingRecords } = saveResults;
newRealEstates.push(...newRecords);
for (const existingRecord of existingRecords) {
const { publishedDate, renewedDate } = existingRecord;
const publishedDateMoment = moment.utc(publishedDate);
const renewedDateMoment = moment.utc(renewedDate);
const stopCrawlingThisCategory = publishedDateMoment.isSame(
renewedDateMoment,
"minute"
);
if (stopCrawlingThisCategory && !OLX_FORCE_CRAWL) {
generatorsToRemove[index] = true;
// console.log("\tGenerator ", index + 1, "has no more new ads");
break;
}
}
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;
// console.log("Generator ", index + 1, "has no more pages");
}
}
// console.log("Generators state : ", generatorsToRemove);
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
if (generatorsToRemove[i]) {
// console.log("\tRemove generator ", i + 1);
indexGenerators.splice(i, 1);
}
}
if (indexGenerators.length === 0) {
done = true;
}
await this.sleep(this.delayBetweenPages);
}
}
return newRealEstates;
}
async *categoryIndexer(adCategory) {
let pageToIndex = 1;
const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
while (true) {
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
const singlePageResults = await this.indexSinglePage(
urlPageToCrawl,
this.maxResultsPerPage
);
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
yield singlePageResults;
} else {
return undefined;
}
++pageToIndex;
if (pageToIndex === this.maxPages) {
return undefined;
}
}
} else {
return undefined;
}
}
async indexSinglePage(url, maxResultsPerPage) {
if (PRINT_CRAWLER_DEBUG) {
console.log("[OLX] Index page : ", url);
}
try {
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
let hrefs = [];
$("#rezultatipretrage")
.find(".listitem")
.each((i, elem) => {
const href = $(elem)
.find("a")
.first()
.attr("href");
if (href) {
hrefs.push(href);
}
});
let actualNoOfResults =
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefs[i]));
}
const scrapedData = await Promise.all(asyncScraping);
const filteredScrapedData = scrapedData.filter(adData => !!adData);
return filteredScrapedData;
} catch (e) {
console.error("Exception caught:" + e);
return [];
}
}
async scrapeAd(url) {
// console.log("Scraping : ", url);
try {
const adPageSource = await fetch(url);
const body = await adPageSource.text();
const $ = cheerio.load(body);
let status = AD_STATUS.STATUS_NORMAL;
const propertySelectors = {
username:
"#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
title: "#naslovartikla",
descriptions: ".artikal_detaljniopis_tekst",
category:
"#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
};
const username = $(propertySelectors.username)
.text()
.trim();
if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
return null;
}
const title = $(propertySelectors.title)
.text()
.trim();
const descriptions = $(propertySelectors.descriptions);
const category = $(propertySelectors.category)
.text()
.trim();
//====== PRICE DETECTION AND EXTRACTION =====
let price = null;
const normalPriceValue = $("#pc > p:nth-child(2)").text();
const urgentPriceValue = $(
"#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
)
.text()
.trim();
if (normalPriceValue && normalPriceValue.length > 0) {
price = normalPriceValue;
if (
$("#pc > p.n")
.text()
.indexOf("Hitna") !== -1
) {
status = AD_STATUS.STATUS_URGENT;
} else {
status = AD_STATUS.STATUS_NORMAL;
}
} else if (urgentPriceValue && urgentPriceValue.length > 0) {
const priceValues = urgentPriceValue.split("KM");
//priceValues will contain values like ["100000", "90000", ...], second element is urgent price
if (priceValues.length > 1) {
price = priceValues[1].trim();
status = AD_STATUS.STATUS_DISCOUNTED;
} else {
throw { message: "Can't find urgent price" };
}
} else {
throw {
message: "Can't find price (it is not normal nor urgent price ?)"
};
}
//====== OTHER AD INFORMATION ===============
let adType = null;
let olxId = null;
let numberOfViewsAgency = null;
let otherInformationDivId;
//We need to locate DIV ID where other information are stored
for (let possibleId = 10; possibleId <= 20; possibleId++) {
const adTypeFieldTitle = $(
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
)
.text()
.trim();
if (adTypeFieldTitle === "Vrsta oglasa") {
otherInformationDivId = possibleId;
break;
}
}
if (!otherInformationDivId) {
throw { message: "Other information DIV could not be found" };
}
const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
const publishedDate = $(publishedDateValueSelector)
.text()
.trim();
const publishedDateMoment = moment.tz(
publishedDate,
OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!publishedDateMoment.isValid()) {
throw { message: "Invalid published date ! Check parsing format" };
}
const renewedDate = $(renewedDateFullValueSelector)
.data("content")
.trim();
const renewedDateMoment = moment.tz(
renewedDate,
OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!renewedDateMoment) {
throw {
message:
"Invalid renewed date ! Check how parser parsed renewed date text"
};
}
adType = $(
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
)
.text()
.trim();
const parsedCategory = this.getAdCategoryId(category);
if (!parsedCategory) {
throw { message: `Unknown ad category [${category}]` };
}
const parsedAdType = this.getAdTypeId(adType);
if (!parsedAdType) {
throw { message: "Unknown ad type" };
}
const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
.text()
.trim();
olxId = $(`${olxIdFieldSelector} > div.df2`)
.text()
.trim();
numberOfViewsAgency = parseInt(
$(numberOfViewsAgencyValueSelector)
.text()
.trim()
);
if (olxIdFieldTitle !== "OLX ID") {
throw { message: "Cannot find correct OLX ID" };
}
//===========================================
//====== DETAIL INFORMATION FIELDS ==========
let area,
gardenSize,
numberOfRooms = null,
numberOfFloors = null,
floor = null,
accessRoadType = null,
heatingType = null,
furnishingType = null,
balcony = null,
newBuilding = null,
elevator = null,
water = null,
electricity = null,
drainageSystem = null,
registeredInZkBooks = null,
recentlyAdapted = null,
parking = null,
garage = null,
gas = null,
antiTheftDoor = null,
airCondition = null,
phoneConnection = null,
cableTV = null,
internet = null,
basementAttic = null,
storeRoom = null,
videoSurveillance = null,
alarm = null,
suitableForStudents = null,
includingBills = null,
animalsAllowed = null,
pool = null,
urbanPlanPermit = null,
buildingPermit = null,
utilityConnection = null,
distanceToRiver = null;
let fieldIndex = 1;
do {
const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
const fieldTitleSelector = `${fieldSelector} > div.df1`;
const fieldValueSelector = `${fieldSelector} > div.df2`;
const fieldTitle = $(fieldTitleSelector)
.text()
.trim()
.toLowerCase();
const fieldValue = $(fieldValueSelector)
.text()
.trim()
.toLowerCase();
switch (fieldTitle) {
case "kvadrata":
area = fieldValue;
break;
case "okućnica (kvadratura)":
gardenSize = fieldValue;
break;
case "broj soba":
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
break;
case "broj prostorija":
numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
break;
case "broj spratova":
numberOfFloors = this.parseNumberOfFloors(
fieldValue,
parsedCategory
);
break;
case "sprat":
floor = this.parseFloorNumber(fieldValue, parsedCategory);
break;
case "vrsta grijanja":
heatingType = this.getHeatingTypeId(fieldValue);
break;
case "namješten?":
furnishingType = this.getFurnishingTypeId(fieldValue);
break;
case "namješten":
furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case "namještena":
furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case "voda":
water = true;
break;
case "struja":
electricity = true;
break;
case "kanalizacija":
drainageSystem = fieldValue !== "nema";
break;
case "godina izgradnje":
newBuilding = newBuilding || fieldValue === "novogradnja";
break;
case "kućni ljubimci":
animalsAllowed = fieldValue === "da";
break;
case "uknjiženo / zk":
registeredInZkBooks = true;
break;
case "uknjiženo (zk)":
registeredInZkBooks = true;
break;
case "novogradnja":
newBuilding = true;
break;
case "nedavno adaptiran":
recentlyAdapted = true;
break;
case "nedavno adaptirana":
recentlyAdapted = true;
break;
case "balkon":
balcony = true;
break;
case "lift":
elevator = true;
break;
case "parking":
parking = true;
break;
case "garaža":
garage = true;
break;
case "plin":
gas = true;
break;
case "blindirana vrata":
antiTheftDoor = true;
break;
case "klima":
airCondition = true;
break;
case "telefonski priključak":
phoneConnection = true;
break;
case "kablovska tv":
cableTV = true;
break;
case "internet":
internet = true;
break;
case "podrum/tavan":
basementAttic = true;
break;
case "ostava/špajz":
storeRoom = true;
break;
case "video nadzor":
videoSurveillance = true;
break;
case "alarm":
alarm = true;
break;
case "za studente":
suitableForStudents = true;
break;
case "uključen trošak režija":
includingBills = true;
break;
case "građevinska dozvola":
buildingPermit = true;
break;
case "komunalni priključak":
utilityConnection = true;
break;
case "urbanistička dozvola":
urbanPlanPermit = true;
break;
case "udaljenost od rijeke (m)":
distanceToRiver = parseInt(fieldValue) || null;
break;
case "prilaz":
accessRoadType = this.getAccessRoadTypeId(fieldValue);
break;
case "bazen":
pool = true;
break;
case "iznajmljeno":
status = AD_STATUS.STATUS_RENTED;
break;
default:
// console.log(fieldTitle, " = ", fieldValue);
break;
}
if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
break;
}
} while (true);
//===========================================
//=========================================
const parsedArea = this.parseArea(area) || null;
const parsedGardenSize = this.parseArea(gardenSize) || null;
const parsedPrice = this.parsePrice(price) || null;
const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
const locationLatLngMatches = latLngRegex.exec(body);
let locationLat = null;
let locationLong = null;
if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
locationLat = parseFloat(locationLatLngMatches[1]) || null;
locationLong = parseFloat(locationLatLngMatches[2]) || null;
}
if (
title.indexOf("[PRODANO]") !== -1 ||
title.indexOf("[ZAVRŠENO]") !== -1
) {
status = AD_STATUS.STATUS_SOLD;
}
const data = {
url,
agencyObjectId: olxId,
originAgencyName: AD_AGENCY.OLX,
realEstateType: parsedCategory,
adType: parsedAdType,
title,
price: parsedPrice,
area: parsedArea,
gardenSize: parsedGardenSize,
shortDescription: descriptions
.first()
.text()
.trim(),
longDescription: descriptions
.last()
.text()
.trim(),
streetNumber: 0,
streetName: "",
locality: "",
municipality: "",
city: "",
region: "",
entity: "",
country: "",
locationLat,
locationLong,
adStatus: status,
publishedDate: publishedDateMoment.toISOString(),
renewedDate: renewedDateMoment.toISOString(),
numberOfRooms,
numberOfFloors,
floor,
accessRoadType,
heatingType,
furnishingType,
balcony,
newBuilding,
elevator,
water,
electricity,
drainageSystem,
registeredInZkBooks,
recentlyAdapted,
parking,
garage,
gas,
antiTheftDoor,
airCondition,
phoneConnection,
cableTV,
internet,
basementAttic,
storeRoom,
videoSurveillance,
alarm,
suitableForStudents,
includingBills,
animalsAllowed,
pool,
urbanPlanPermit,
buildingPermit,
utilityConnection,
distanceToRiver,
numberOfViewsAgency
};
return data;
} catch (e) {
console.error("Exception caught: " + e.message, "\r\nURL:", url);
}
return null;
}
//======= HELPER FUNCTIONS =============
getAdCategoryId(categoryText) {
switch (categoryText) {
case "Stanovi":
return AD_CATEGORY.FLAT.id;
case "Zemljišta":
return AD_CATEGORY.LAND.id;
case "Kuće":
return AD_CATEGORY.HOUSE.id;
case "Poslovni prostori":
return AD_CATEGORY.OFFICE.id;
case "Apartmani":
return AD_CATEGORY.APARTMENT.id;
case "Garaže":
return AD_CATEGORY.GARAGE.id;
case "Vikendice":
return AD_CATEGORY.COTTAGE.id;
default:
return undefined;
}
}
getAdTypeId(adTypeText) {
switch (adTypeText) {
case "Prodaja":
return AD_TYPE.AD_TYPE_SALE.stringId;
case "Izdavanje":
return AD_TYPE.AD_TYPE_RENT.stringId;
case "Potražnja":
return AD_TYPE.AD_TYPE_REQUEST.stringId;
default:
return undefined;
}
}
getHeatingTypeId(heatingTypeText) {
switch (heatingTypeText) {
case "struja":
return HEATING_TYPE.ELECTRICITY.id;
case "plin":
return HEATING_TYPE.GAS.id;
case "drva":
return HEATING_TYPE.WOOD.id;
case "centralno (gradsko)":
return HEATING_TYPE.CENTRAL_CITY.id;
case "centralno (kotlovnica)":
return HEATING_TYPE.CENTRAL_BOILER.id;
case "centralno (plin)":
return HEATING_TYPE.CENTRAL_GAS.id;
case "nije uvedeno":
return HEATING_TYPE.NO_HEATING.id;
case "ostalo":
return HEATING_TYPE.OTHER.id;
case "drugo":
return HEATING_TYPE.OTHER.id;
default:
console.log("grijanje = NEPOZNATO [", heatingTypeText, "]");
return null;
}
}
getFurnishingTypeId(furnishingTypeText) {
switch (furnishingTypeText) {
case "namješten":
return FURNISHING_TYPE.FURNISHED.id;
case "polunamješten":
return FURNISHING_TYPE.HALF_FURNISHED.id;
case "nenamješten":
return FURNISHING_TYPE.NOT_FURNISHED.id;
case "":
return FURNISHING_TYPE.FURNISHED.id;
default:
console.log("namješten = NEPOZNATO [", furnishingTypeText, "]");
return null;
}
}
getAccessRoadTypeId(accessRoadTypeText) {
switch (accessRoadTypeText) {
case "asfalt":
return ACCESS_ROAD_TYPE.ASPHALT.id;
case "beton":
return ACCESS_ROAD_TYPE.CONCRETE.id;
case "makadam":
return ACCESS_ROAD_TYPE.MACADAM.id;
case "ostalo":
return ACCESS_ROAD_TYPE.OTHER.id;
default:
console.log("pristup = NEPOZNATO [", accessRoadTypeText, "]");
return null;
}
}
parseArea(areaText) {
if (!areaText) {
return NaN;
}
const removeDotsExceptLastOneRegex = /[.](?=.*[.])/g;
const textWithOnlyOneDecimalDot = areaText
.replace(",", ".")
.replace(removeDotsExceptLastOneRegex, "");
return parseFloat(textWithOnlyOneDecimalDot);
}
parsePrice(priceText) {
if (!priceText) {
return NaN;
}
const formattedPriceText = priceText.replace(".", "").replace(",", ".");
return parseFloat(formattedPriceText);
}
parseNumberOfRooms(numberOfRoomsText, categoryId) {
if (categoryId === AD_CATEGORY.FLAT.id) {
switch (numberOfRoomsText) {
case "garsonjera":
return 0;
case "jednosoban (1)":
return 1;
case "jednoiposoban (1.5)":
return 1.5;
case "dvosoban (2)":
return 2;
case "trosoban (3)":
return 3;
case "četverosoban (4)":
return 4;
case "petosoban i više":
return 5;
default:
console.log(
"broj soba [stan] = NEPOZNATO [",
numberOfRoomsText,
", ",
categoryId,
"]"
);
return null;
}
}
if (
categoryId === AD_CATEGORY.HOUSE.id ||
categoryId === AD_CATEGORY.COTTAGE.id ||
categoryId === AD_CATEGORY.APARTMENT.id ||
categoryId === AD_CATEGORY.OFFICE.id
) {
return parseInt(numberOfRoomsText) || null;
}
console.log("broj soba = NEPOZNATO [", numberOfRoomsText, "]");
return null;
}
parseNumberOfFloors(numberOfFloorsText, categoryId) {
if (
categoryId === AD_CATEGORY.HOUSE.id ||
categoryId === AD_CATEGORY.COTTAGE.id
) {
return parseInt(numberOfFloorsText) || null;
}
if (categoryId === AD_CATEGORY.OFFICE.id) {
if (
numberOfFloorsText === "suteren" ||
numberOfFloorsText === "prizemlje"
) {
return 0;
}
if (numberOfFloorsText === "6+") {
return 7;
}
return parseInt(numberOfFloorsText) || null;
}
console.log("broj spratova = NEPOZNATO [", numberOfFloorsText, "]");
return null;
}
parseFloorNumber(floorText, categoryId) {
if (
categoryId === AD_CATEGORY.FLAT.id ||
categoryId === AD_CATEGORY.APARTMENT.id
) {
if (
floorText === "suteren" ||
floorText === "prizemlje" ||
floorText === "visoko prizemlje"
) {
return 0;
}
return parseInt(floorText) || null;
}
if (categoryId === AD_CATEGORY.OFFICE.id) {
if (floorText === "zaseban objekat") {
return null;
}
if (floorText === "prizemlje" || floorText === "visoko prizemlje") {
return 0;
}
return parseInt(floorText) || null;
}
console.log("sprat = NEPOZNATO [", floorText, "]");
return null;
}
async sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async saveCrawledResults(results) {
const savers = this.savers;
// for (const saver of savers) {
// await saver.save(results);
// }
//For now, we use only Postgres saver, so ...
return savers[0].save(results);
//so that we can use some sequelize options and information when data is inserted
}
}
module.exports = OlxCrawler;

View File

@@ -1,574 +0,0 @@
"use strict";
const fetch = require("node-fetch");
const cheerio = require("cheerio");
const moment = require("moment-timezone");
const {
AD_TYPE,
AD_CATEGORY,
AD_AGENCY,
AD_STATUS,
CRAWLER_AD_TYPE,
FURNISHING_TYPE,
HEATING_TYPE
} = require("../../common/enums");
const {
PRINT_CRAWLER_DEBUG,
DEFAULT_TIMEZONE
} = require("../../config/appConfig");
const { PROSTOR_FORCE_CRAWL } = require("../specificConfigs/prostor");
const PROSTOR_ENUMS = {
PROSTOR_AD_TYPE: {
[CRAWLER_AD_TYPE.ALL]: "&action=0",
[CRAWLER_AD_TYPE.ONLY_SELL]: "&action=1",
[CRAWLER_AD_TYPE.ONLY_RENT]: "&action=2"
},
PROSTOR_AD_CATEGORY: {
[AD_CATEGORY.ALL.id]: "",
[AD_CATEGORY.FLAT.id]: "&type=7",
[AD_CATEGORY.HOUSE.id]: "&type=8",
[AD_CATEGORY.LAND.id]: "&type=10",
[AD_CATEGORY.OFFICE.id]: "&type=9",
[AD_CATEGORY.APARTMENT.id]: "&type=11",
[AD_CATEGORY.GARAGE.id]: "&type=14"
//[AD_CATEGORY.COTTAGE.id]: ""
},
PROSTOR_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
PROSTOR_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
};
class ProstorCrawler {
constructor(
savers = [],
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
maxPages = 5000,
maxResultsPerPage = 5000,
ignoredUsernames = [],
delayBetweenPages = 1000
) {
this.savers = savers;
this.baseUrl = "https://prostor.ba/pretraga";
this.crawlerAdTypes = crawlerAdTypes;
this.crawlerAdCategories = crawlerAdCategories;
this.maxResultsPerPage = maxResultsPerPage;
this.delayBetweenPages = delayBetweenPages;
}
async crawl() {
const crawlAdCategories = this.crawlerAdCategories;
const newRealEstates = [];
if (crawlAdCategories) {
const indexGenerators = [];
for (const adCategory of crawlAdCategories) {
indexGenerators.push(this.categoryIndexer(adCategory));
}
let done = false;
while (!done) {
const categoryIndexerPromises = [];
const generatorsToRemove = [];
for (const indexGenerator of indexGenerators) {
categoryIndexerPromises.push(indexGenerator.next());
generatorsToRemove.push(false);
}
const singlePageResults = await Promise.all(categoryIndexerPromises);
const entries = singlePageResults.entries();
for (const [index, { value: singlePageResult }] of entries) {
if (singlePageResult) {
const saveResults = await this.saveCrawledResults(singlePageResult);
const { newRecords } = saveResults;
newRealEstates.push(...newRecords);
if (
Array.isArray(newRecords) &&
newRecords.length === 0 &&
!PROSTOR_FORCE_CRAWL
) {
generatorsToRemove[index] = true;
}
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;
// console.log("Generator ", index + 1, "has no more pages");
}
}
// console.log("Generators state : ", generatorsToRemove);
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
if (generatorsToRemove[i]) {
// console.log("\tRemove generator ", i + 1);
indexGenerators.splice(i, 1);
}
}
if (indexGenerators.length === 0) {
done = true;
}
await this.sleep(this.delayBetweenPages);
}
}
return newRealEstates;
}
async *categoryIndexer(adCategory) {
const urlAdTypePart = PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
const urlPageToCrawl = `${this.baseUrl}?remove_sold=0${urlAdTypePart}${urlCategoryPart}`;
const listOfAllRealEstates = await this.extractRealEstates(
urlPageToCrawl
);
let elementToStartIndexFrom = 0;
while (true) {
const realEstatesForSinglePage = listOfAllRealEstates.slice(
elementToStartIndexFrom,
elementToStartIndexFrom + this.maxResultsPerPage
);
if (realEstatesForSinglePage.length > 0) {
elementToStartIndexFrom += realEstatesForSinglePage.length;
const singlePageResults = await this.indexSinglePage(
realEstatesForSinglePage
);
const filteredSinglePageResults = singlePageResults.filter(
singleResult => !!singleResult
);
if (
Array.isArray(filteredSinglePageResults) &&
filteredSinglePageResults.length > 0
) {
yield filteredSinglePageResults;
} else {
return undefined;
}
} else {
return undefined;
}
}
} else {
return undefined;
}
}
async indexSinglePage(realEstatesList) {
const asyncActions = [];
for (const realEstate of realEstatesList) {
asyncActions.push(this.scrapeAd(realEstate));
}
try {
return await Promise.all(asyncActions);
} catch (e) {
console.log(
"[PROSTOR] Error crawling ads : ",
e.message || "UNKNOWN ERROR"
);
return [];
}
}
async scrapeAd(realEstate) {
const { lat, lng, property_name, price, size, link, status } = realEstate;
const url = `https://prostor.ba${link}`;
// console.log("[PROSTOR] Scraping : ", url);
try {
const adPageSource = await fetch(url);
const body = await adPageSource.text();
const $ = cheerio.load(body);
// link contains part of the URL in the format of : /prodaja/stan/stup/9556
// general form is : /actionType/realEstateType/location/realEstateID
// linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
const linkParts = link.split("/");
const adType = ProstorCrawler.getAdTypeId(linkParts[1]);
const realEstateType = ProstorCrawler.getAdCategoryId(linkParts[2]);
const prostorId = linkParts[4];
if (!adType || !realEstateType || !prostorId) {
return null;
}
const allDataSelector =
"body > div > div.container-fluid > div > div.column-right > table > tbody";
const realEstateProperties = {};
$(allDataSelector)
.find("p")
.each((i, element) => {
const propertyElement = $(element)
.text()
.split(":")
.map(text => text.trim().toLowerCase());
const propertyTitle = propertyElement[0];
realEstateProperties[propertyTitle] = propertyElement[1];
});
$(allDataSelector)
.find("div.mb-2")
.each((i, element) => {
const propertyElement = $(element)
.text()
.trim()
.toLowerCase();
realEstateProperties[propertyElement] = true;
});
if (JSON.stringify(realEstateProperties) === JSON.stringify({})) {
return null;
}
let numberOfRooms =
parseFloat(realEstateProperties["broj soba"]) +
parseFloat(realEstateProperties["broj spavaćih soba"]) || null,
numberOfFloors = null,
floor = null,
accessRoadType = null,
heatingType = ProstorCrawler.getHeatingTypeId(realEstateProperties),
furnishingType = null,
balcony =
realEstateProperties["balkon"] ||
realEstateProperties["terasa"] ||
realEstateProperties["lođa"] ||
null,
newBuilding = linkParts[1] === "novogradnja",
elevator = realEstateProperties["lift"] || null,
water = realEstateProperties["voda"] || null,
electricity = realEstateProperties["električna energija"] || null,
drainageSystem = realEstateProperties["kanalizacija"] || null,
registeredInZkBooks = null,
recentlyAdapted = null,
parking = realEstateProperties["parking"] || null,
garage = realEstateProperties["garaža"] || null,
gas = realEstateProperties["plin"] || null,
antiTheftDoor = realEstateProperties["blindo vrata"] || null,
airCondition = realEstateProperties["klima"] || null,
phoneConnection = realEstateProperties["telefon"] || null,
cableTV = realEstateProperties["kablovksa tv"] || null,
internet =
realEstateProperties["internet"] ||
realEstateProperties["adsl"] ||
null,
basementAttic = realEstateProperties["podrum"] || null,
storeRoom = realEstateProperties["ostava"] || null,
videoSurveillance = realEstateProperties["video nadzor"],
alarm = realEstateProperties["alarm"] || null,
suitableForStudents = null,
includingBills = null,
animalsAllowed = null,
pool = realEstateProperties["bazen"] || null,
urbanPlanPermit = null,
buildingPermit = null,
utilityConnection = null,
distanceToRiver = null,
numberOfViewsAgency = null;
// Floor versions (there are possibly more versions) :
// Sprat: 3/3
// Sprat: 1 - 2/2
// Sprat: Pr - 7/7
// Sprat: -2/0
// If there are two parts, that represents more real estates are sold
// numberOfFloors is contained in second part, after / sign
const floorsArray = realEstateProperties["sprat"].split(" - ");
let floorText = "";
if (floorsArray.length === 1) {
const floorDescription = floorsArray[0].split("/");
numberOfFloors = parseInt(floorDescription[1]) || null;
floorText = floorDescription[0];
floor = Math.round(parseFloat(floorText));
} else if (floorsArray.length === 2) {
const floorDescription = floorsArray[1].split("/");
numberOfFloors = parseInt(floorDescription[1]) || null;
floorText = floorsArray[0];
floor = Math.round(parseFloat(floorText));
} else {
// This is something strange
}
if (isNaN(floor)) {
// It was textual representation of floor, like "Pr", "Su" or similar
switch (floorText) {
case "pr":
floor = 0;
break;
case "su":
floor = -1;
break;
default:
console.log(
"[PROSTOR] Unknown textual representation of floor : ",
floorText
);
floor = null;
}
}
if (realEstateProperties["namješteno"]) {
furnishingType = FURNISHING_TYPE.FURNISHED.id;
} else if (realEstateProperties["polunamješteno"]) {
furnishingType = FURNISHING_TYPE.HALF_FURNISHED.id;
} else {
furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
}
const adStatus = ProstorCrawler.getStatusId(status);
const title = property_name;
const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
const parsedArea = parseFloat(size);
const gardenSize = null;
const longDescription = null;
const data = {
url,
agencyObjectId: prostorId,
originAgencyName: AD_AGENCY.PROSTOR,
realEstateType,
adType,
title,
price: parsedPrice,
area: parsedArea,
gardenSize,
shortDescription: "",
longDescription: longDescription,
streetNumber: 0,
streetName: realEstateProperties["adresa"],
locality: "",
municipality: "",
city: "",
region: "",
entity: "",
country: "",
locationLat: lat,
locationLong: lng,
adStatus,
numberOfRooms,
numberOfFloors,
floor,
accessRoadType,
heatingType,
furnishingType,
balcony,
newBuilding,
elevator,
water,
electricity,
drainageSystem,
registeredInZkBooks,
recentlyAdapted,
parking,
garage,
gas,
antiTheftDoor,
airCondition,
phoneConnection,
cableTV,
internet,
basementAttic,
storeRoom,
videoSurveillance,
alarm,
suitableForStudents,
includingBills,
animalsAllowed,
pool,
urbanPlanPermit,
buildingPermit,
utilityConnection,
distanceToRiver,
numberOfViewsAgency
};
return data;
} catch (e) {
console.error(
"[PROSTOR] Exception caught: " + e.message,
"\r\nURL:",
url
);
return null;
}
}
async extractRealEstates(url) {
if (PRINT_CRAWLER_DEBUG) {
console.log("[PROSTOR] Index page : ", url);
}
try {
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
const scriptElement = $(
"body > div > div.container-fluid > script:nth-child(7)"
);
if (
scriptElement[0] &&
scriptElement[0].children &&
scriptElement[0].children[0] &&
scriptElement[0].children[0].data
) {
const scriptData = scriptElement[0].children[0].data;
try {
// script element data contains JS code and we need to extract only data for realEstates
// data string starts with : var map; var markers = [{"r ...
// so we remove first 23 characters
//
// real estate JSON data ends with ...}, ]; map = new...
// so we need to find index of that substring to know where to stop
// we will NOT include trailing comma because it breaks JSON parse, so we have to close ] bracket manually
const jsonEndIndex = scriptData.indexOf(", ]; map = new");
if (jsonEndIndex > -1) {
const jsonData = scriptData.substring(23, jsonEndIndex) + "]";
const realEstates = JSON.parse(jsonData);
// const transformedRealEstates = [];
//
// for (const realEstate of realEstates) {
// const transformedRealEstate = ProstorCrawler.transformRealEstateData(
// realEstate
// );
// if (transformedRealEstate) {
// transformedRealEstates.push(transformedRealEstate);
// }
// }
//
// return transformedRealEstates;
return realEstates;
} else {
throw {
message: "Something is wrong with JSON data or data is moved"
};
}
} catch (e) {
console.log(e);
throw e;
}
}
} catch (e) {
console.error(
"[PROSTOR] Exception caught:",
e.message || "UNKNOWN MESSAGE"
);
return [];
}
}
//======= HELPER FUNCTIONS =============
static getAdCategoryId(categoryText) {
switch (categoryText) {
case "stan":
return AD_CATEGORY.FLAT.id;
case "kuca":
return AD_CATEGORY.HOUSE.id;
case "apartman":
return AD_CATEGORY.APARTMENT.id;
case "poslovni-prostor":
return AD_CATEGORY.OFFICE.id;
case "garaza":
return AD_CATEGORY.GARAGE.id;
case "zemljiste":
return AD_CATEGORY.LAND.id;
default:
return undefined;
}
}
static getAdTypeId(adTypeText) {
switch (adTypeText) {
case "prodaja":
return AD_TYPE.AD_TYPE_SALE.stringId;
case "najam":
return AD_TYPE.AD_TYPE_RENT.stringId;
case "novogradnja":
return AD_TYPE.AD_TYPE_SALE.stringId;
default:
return undefined;
}
}
static getHeatingTypeId(realEstateProperties) {
const realEstatePropertiesKeys = Object.keys(realEstateProperties);
for (const property of realEstatePropertiesKeys) {
switch (property) {
case "centralno toplane":
return HEATING_TYPE.CENTRAL_CITY.id;
case "etažno plinsko":
return HEATING_TYPE.CENTRAL_GAS.id;
case "termo blok":
case "podno grijanje":
return HEATING_TYPE.OTHER.id;
case "etažno električno":
case "konvektori":
return HEATING_TYPE.ELECTRICITY.id;
case "plinske peći":
return HEATING_TYPE.GAS.id;
case "vlastita kotlovnica":
return HEATING_TYPE.CENTRAL_BOILER.id;
case "toplotna pumpa":
return HEATING_TYPE.HEAT_PUMP.id;
case "kamin":
return HEATING_TYPE.WOOD.id;
default:
//console.log("[PROSTOR] Nepoznato >>> [", property, "]");
}
}
}
static getStatusId(statusText) {
switch (statusText) {
case "":
return AD_STATUS.STATUS_NORMAL;
case "Rezervisano":
return AD_STATUS.STATUS_RESERVED;
case "Prodano":
return AD_STATUS.STATUS_SOLD;
case "Iznajmljeno":
return AD_STATUS.STATUS_RENTED;
default:
console.log("[PROSTOR] Unknown AD_STATUS : [", statusText, "]");
return AD_STATUS.STATUS_NORMAL;
}
}
async sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async saveCrawledResults(results) {
const savers = this.savers;
// for (const saver of savers) {
// await saver.save(results);
// }
//For now, we use only Postgres saver, so ...
return savers[0].save(results);
//so that we can use some sequelize options and information when data is inserted
}
}
module.exports = ProstorCrawler;

View File

@@ -1,838 +0,0 @@
"use strict";
const fetch = require("node-fetch");
const cheerio = require("cheerio");
const Promise = require("bluebird");
const moment = require("moment-timezone");
const htmlToText = require("html-to-text");
const {
AD_TYPE,
AD_CATEGORY,
AD_AGENCY,
AD_STATUS,
CRAWLER_AD_TYPE,
HEATING_TYPE,
ACCESS_ROAD_TYPE,
FURNISHING_TYPE
} = require("../../common/enums");
const {
DEFAULT_TIMEZONE,
PRINT_CRAWLER_DEBUG
} = require("../../config/appConfig");
const RENTAL_ENUMS = {
RENTAL_AD_TYPE: {
[CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
[CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
[CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
},
RENTAL_AD_CATEGORY: {
[AD_CATEGORY.ALL.id]: "",
[AD_CATEGORY.FLAT.id]: "/tip-2",
[AD_CATEGORY.HOUSE.id]: "/tip-1",
[AD_CATEGORY.LAND.id]: "/tip-5",
[AD_CATEGORY.OFFICE.id]: "/tip-4",
[AD_CATEGORY.APARTMENT.id]: "/tip-3",
[AD_CATEGORY.GARAGE.id]: "/tip-6"
//[AD_CATEGORY.COTTAGE.id]: ""
},
RENTAL_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
};
const { RENTAL_FORCE_CRAWL } = require("../specificConfigs/rental");
class RentalCrawler {
constructor(
savers = [],
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
maxPages = 1000,
maxResultsPerPage = 100,
ignoredUsernames = [],
delayBetweenPages = 1000
) {
this.savers = savers;
this.baseUrl = "https://www.rental.ba/pretraga/sortiraj-date_DESC";
this.crawlerAdTypes = crawlerAdTypes;
this.crawlerAdCategories = crawlerAdCategories;
this.maxPages = maxPages;
this.maxResultsPerPage = maxResultsPerPage;
this.delayBetweenPages = delayBetweenPages;
}
async crawl() {
const crawlAdCategories = this.crawlerAdCategories;
const newRealEstates = [];
if (crawlAdCategories) {
const indexGenerators = [];
for (const adCategory of crawlAdCategories) {
indexGenerators.push(this.categoryIndexer(adCategory));
}
let done = false;
while (!done) {
const categoryIndexerPromises = [];
const generatorsToRemove = [];
for (const indexGenerator of indexGenerators) {
categoryIndexerPromises.push(indexGenerator.next());
generatorsToRemove.push(false);
}
const singlePageResults = await Promise.all(categoryIndexerPromises);
const entries = singlePageResults.entries();
for (const [index, { value: singlePageResult }] of entries) {
if (singlePageResult) {
const saveResults = await this.saveCrawledResults(singlePageResult);
const { newRecords } = saveResults;
newRealEstates.push(...newRecords);
if (
Array.isArray(newRecords) &&
newRecords.length === 0 &&
!RENTAL_FORCE_CRAWL
) {
generatorsToRemove[index] = true;
}
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;
// console.log("Generator ", index + 1, "has no more pages");
}
}
// console.log("Generators state : ", generatorsToRemove);
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
if (generatorsToRemove[i]) {
// console.log("\tRemove generator ", i + 1);
indexGenerators.splice(i, 1);
}
}
if (indexGenerators.length === 0) {
done = true;
}
await this.sleep(this.delayBetweenPages);
}
}
return newRealEstates;
}
async *categoryIndexer(adCategory) {
let pageToIndex = 1;
const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
while (true) {
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
const singlePageResults = await this.indexSinglePage(
urlPageToCrawl,
this.maxResultsPerPage
);
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
yield singlePageResults;
} else {
return undefined;
}
++pageToIndex;
if (pageToIndex === this.maxPages) {
return undefined;
}
}
} else {
return undefined;
}
}
async indexSinglePage(url, maxResultsPerPage) {
if (PRINT_CRAWLER_DEBUG) {
console.log("[RENTAL] Index page : ", url);
}
try {
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
let hrefs = [];
$(
"body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
)
.find(".pull-right")
.each((i, elem) => {
const href = $(elem)
.find("a")
.first()
.attr("href");
if (href) {
hrefs.push(href);
}
});
let actualNoOfResults =
hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefs[i]));
}
const scrapedData = await Promise.all(asyncScraping);
const filteredScrapedData = scrapedData.filter(adData => !!adData);
return filteredScrapedData;
} catch (e) {
console.error("[RENTAL] Exception caught:" + e);
return [];
}
}
async scrapeAd(url) {
// console.log("[RENTAL] Scraping : ", url);
try {
const adPageSource = await fetch(url);
const body = await adPageSource.text();
const $ = cheerio.load(body);
const mapElementParent = $(".box-map").parent();
const scriptElement = $("script", mapElementParent);
if (
scriptElement[0] &&
scriptElement[0].children &&
scriptElement[0].children[0] &&
scriptElement[0].children[0].data
) {
let extractedData;
try {
//data string starts with : var json_map_data = [{"r ...
//so we remove first 20 characters
const jsonData = scriptElement[0].children[0].data.substring(20);
const parsedJsonData = JSON.parse(jsonData);
extractedData = parsedJsonData[0];
} catch (e) {
throw { message: "Can't find ad data JSON" };
}
let adStatus = AD_STATUS.STATUS_NORMAL;
const rentalId = extractedData["re_realEstates_id"];
const adCategory = this.getKiviCategoryIdFromRentalId(
parseInt(extractedData["re_types_id"])
);
if (!adCategory) {
throw {
message: `Invalid category : ${extractedData["re_types_id"]}`
};
}
const adType = this.getKiviAdTypeFromRentalActionId(
parseInt(extractedData["re_action_id"])
);
if (!adType) {
throw {
message: `Invalid ad type : ${extractedData["re_action_id"]}`
};
}
const descriptionIds = extractedData["re_descriptions_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(descriptionIds)) {
throw {
message:
'Expected array od descriptions but "re_descriptions_id" not found !'
};
}
const spaceIds = extractedData["re_spaces_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(spaceIds)) {
throw {
message: 'Expected array od spaces but "re_spaces_id" not found !'
};
}
const infrastructureIds = extractedData["re_infrastructure_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(infrastructureIds)) {
throw {
message:
'Expected array od infrastructures but "re_infrastructure_id" not found !'
};
}
const floorNoIds = extractedData["re_floorNO_id"]
.split(",")
.map(stringNumber => parseInt(stringNumber));
if (!Array.isArray(floorNoIds)) {
throw {
message:
'Expected array od infrastructures but "re_floorNO_id" not found !'
};
}
const numberOfViewsAgencySelector = $(
"body > div > div.container > div.row.content-top > div.col-xs-12.col-sm-12.col-md-9 > div > div.box-viewcount"
);
// number of views is written as : "Broj pregledavanja: NNN"
const numberOfViewsAgencyFullText = numberOfViewsAgencySelector
.text()
.trim();
const numberOfViewsAgencyParts = numberOfViewsAgencyFullText.split(":");
const realEstatePropertiesFromDescriptions = this.getPropertiesFromDescriptions(
descriptionIds
);
const realEstatePropertiesFromSpaces = this.getPropertiesFromSpaces(
spaceIds
);
const realEstatePropertiesFromInfrastructure = this.getPropertiesFromInfrastructure(
infrastructureIds
);
if (extractedData["adm_realEstates_discount"] === "1") {
adStatus = AD_STATUS.STATUS_DISCOUNTED;
}
let numberOfRooms =
parseInt(extractedData["re_realEstates_roomsNO"]) +
parseInt(extractedData["re_realEstates_bedNO"]) || null,
numberOfFloors =
parseInt(extractedData["re_realEstates_floorsNO"]) ||
this.getNumberOfFloorsFromFloorId(extractedData["re_floorNO_id"]),
floor =
parseInt(extractedData["re_realEstates_floorNO"]) ||
this.getFloorNumberFromFloorId(extractedData["re_floorNO_id"]),
accessRoadType = realEstatePropertiesFromDescriptions.accessRoadType,
heatingType =
this.getHeatingTypeId(extractedData["re_heating_id"]) || null,
furnishingType = realEstatePropertiesFromDescriptions.furnishingType,
balcony =
realEstatePropertiesFromDescriptions.balcony ||
realEstatePropertiesFromSpaces.balcony,
newBuilding = extractedData["op_realEstates_newBuilding"]
? extractedData["op_realEstates_newBuilding"] === "1"
: null,
elevator = realEstatePropertiesFromDescriptions.elevator,
water =
realEstatePropertiesFromDescriptions.water ||
realEstatePropertiesFromInfrastructure.water,
electricity =
realEstatePropertiesFromDescriptions.electricity ||
realEstatePropertiesFromInfrastructure.electricity,
drainageSystem =
realEstatePropertiesFromInfrastructure.drainageSystem,
registeredInZkBooks =
extractedData["op_realEstates_ownerPermit"] === 1 || null,
recentlyAdapted = null,
parking =
realEstatePropertiesFromDescriptions.parking ||
realEstatePropertiesFromSpaces.parking,
garage = realEstatePropertiesFromSpaces.garage,
gas = realEstatePropertiesFromInfrastructure.gas,
antiTheftDoor = realEstatePropertiesFromDescriptions.antiTheftDoor,
airCondition = realEstatePropertiesFromDescriptions.airCondition,
phoneConnection =
realEstatePropertiesFromInfrastructure.phoneConnection,
cableTV = realEstatePropertiesFromInfrastructure.cableTV,
internet = realEstatePropertiesFromInfrastructure.internet,
basementAttic =
realEstatePropertiesFromSpaces.basementAttic ||
this.checkBasemAtticFromFloors(extractedData["re_floorNO_id"]),
storeRoom = realEstatePropertiesFromSpaces.storeRoom,
videoSurveillance =
realEstatePropertiesFromDescriptions.videoSurveillance ||
realEstatePropertiesFromInfrastructure.videoSurveillance,
alarm = realEstatePropertiesFromDescriptions.alarm,
suitableForStudents = null,
includingBills =
extractedData["op_realEstates_utilitiesIncluded"] === "1" || null,
animalsAllowed = null,
pool = realEstatePropertiesFromDescriptions.pool,
urbanPlanPermit =
extractedData["op_realEstates_locationPermit"] === "1" ||
realEstatePropertiesFromDescriptions.urbanPlanPermit,
buildingPermit =
extractedData["op_realEstates_buildingPermit"] === "1" || null,
utilityConnection =
realEstatePropertiesFromDescriptions.utilityConnection,
distanceToRiver = null,
numberOfViewsAgency =
numberOfViewsAgencyParts.length > 1
? parseInt(numberOfViewsAgencyParts[1])
: null;
const title = extractedData["re_realEstates_portalName"];
const extractedPrice = parseFloat(
extractedData["re_realEstates_price"]
);
const price = extractedPrice ? extractedPrice : null;
const area = parseFloat(extractedData["re_realEstates_area"]);
const gardenSize = parseFloat(
extractedData["re_realEstates_fieldArea"]
);
const longDescription = htmlToText.fromString(
extractedData["re_realEstates_description"]
);
const locationLong = extractedData["re_realEstates_longitude"];
const locationLat = extractedData["re_realEstates_latitude"];
const publishedDateMoment = moment.tz(
extractedData["re_realEstates_inserted"],
RENTAL_ENUMS.RENTAL_PUBLISHED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!publishedDateMoment.isValid()) {
throw {
message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
};
}
const renewedDateMoment = moment.tz(
extractedData["re_realEstates_edited"],
RENTAL_ENUMS.RENTAL_RENEWED_DATE_FORMAT,
DEFAULT_TIMEZONE
);
if (!renewedDateMoment.isValid()) {
throw {
message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
};
}
const data = {
url,
agencyObjectId: rentalId,
originAgencyName: AD_AGENCY.RENTAL,
realEstateType: adCategory,
adType,
title,
price,
area,
gardenSize,
shortDescription: "",
longDescription: longDescription,
streetNumber: 0,
streetName: "",
locality: "",
municipality: "",
city: "",
region: "",
entity: "",
country: "",
locationLat,
locationLong,
adStatus,
publishedDate: publishedDateMoment.toISOString(),
renewedDate: renewedDateMoment.toISOString(),
numberOfRooms,
numberOfFloors,
floor,
accessRoadType,
heatingType,
furnishingType,
balcony,
newBuilding,
elevator,
water,
electricity,
drainageSystem,
registeredInZkBooks,
recentlyAdapted,
parking,
garage,
gas,
antiTheftDoor,
airCondition,
phoneConnection,
cableTV,
internet,
basementAttic,
storeRoom,
videoSurveillance,
alarm,
suitableForStudents,
includingBills,
animalsAllowed,
pool,
urbanPlanPermit,
buildingPermit,
utilityConnection,
distanceToRiver,
numberOfViewsAgency
};
return data;
} else {
console.log("[RENTAL] No JSON data for this ad : ", url);
return null;
}
} catch (e) {
console.error("[RENTAL] Exception caught: " + e.message, "\r\nURL:", url);
return null;
}
return null;
}
//======= HELPER FUNCTIONS =============
getKiviCategoryIdFromRentalId(rentalCategoryId) {
switch (rentalCategoryId) {
case 1:
return AD_CATEGORY.HOUSE.id;
case 2:
return AD_CATEGORY.FLAT.id;
case 3:
return AD_CATEGORY.APARTMENT.id;
case 4:
return AD_CATEGORY.OFFICE.id;
case 5:
return AD_CATEGORY.LAND.id;
case 6:
return AD_CATEGORY.GARAGE.id;
default:
return undefined;
}
}
getKiviAdTypeFromRentalActionId(actionId) {
switch (actionId) {
case 1:
return AD_TYPE.AD_TYPE_SALE.stringId;
case 2:
return AD_TYPE.AD_TYPE_RENT.stringId;
default:
return undefined;
}
}
getPropertiesFromDescriptions(descriptionIds) {
const result = {
accessRoadType: null,
furnishingType: null,
balcony: null,
elevator: null,
parking: null,
antiTheftDoor: null,
airCondition: null,
videoSurveillance: null,
alarm: null,
pool: null,
urbanPlanPermit: null,
utilityConnection: null,
water: null,
electricity: null
};
for (const descriptionId of descriptionIds) {
switch (descriptionId) {
case 16:
result.furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
break;
case 17:
result.furnishingType = FURNISHING_TYPE.HALF_FURNISHED.id;
break;
case 1:
case 28:
result.furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case 14:
result.elevator = true;
break;
case 39:
result.electricity = true;
break;
case 40:
result.water = true;
break;
case 41:
case 58:
result.accessRoadType = ACCESS_ROAD_TYPE.ASPHALT.id;
break;
case 26:
result.balcony = true;
break;
case 62:
result.parking = true;
break;
case 3:
result.antiTheftDoor = true;
break;
case 2:
case 21:
result.airCondition = true;
break;
case 4:
result.alarm = true;
break;
case 55:
result.videoSurveillance = true;
break;
case 9:
result.pool = true;
break;
case 60:
result.urbanPlanPermit = true;
break;
case 38:
result.utilityConnection = true;
break;
}
}
return result;
}
getPropertiesFromSpaces(spaceIds) {
const result = {
balcony: null,
parking: null,
garage: null,
basementAttic: null,
storeRoom: null
};
for (const spaceId of spaceIds) {
switch (spaceId) {
case 36:
case 12:
result.parking = true;
break;
case 1:
case 2:
case 3:
result.balcony = true;
break;
case 4:
case 30:
result.garage = true;
break;
case 9:
case 10:
result.storeRoom = true;
break;
case 18:
case 34:
case 37:
case 27:
result.basementAttic = true;
break;
}
}
return result;
}
getHeatingTypeId(heatingRentalId) {
// heatingRentalId can have multiple values, like: "1, 2, 3", parseInt will take first integer value
const heatingId = parseInt(heatingRentalId);
switch (heatingId) {
case 27:
case 16:
return HEATING_TYPE.GAS.id;
case 4:
return HEATING_TYPE.CENTRAL_GAS.id;
case 3:
case 23:
case 6:
case 7:
case 8:
case 9:
case 10:
return HEATING_TYPE.CENTRAL_BOILER.id;
case 2:
case 13:
case 30:
case 17:
case 29:
case 31:
return HEATING_TYPE.ELECTRICITY.id;
case 24:
case 25:
case 12:
return HEATING_TYPE.CENTRAL_CITY.id;
case 26:
case 21:
case 20:
return HEATING_TYPE.WOOD.id;
case 28:
case 19:
return HEATING_TYPE.HEAT_PUMP.id;
case 14:
case 32:
return HEATING_TYPE.OTHER.id;
default:
return null;
}
}
getPropertiesFromInfrastructure(infrastructureIds) {
const result = {
electricity: null,
water: null,
gas: null,
drainageSystem: null,
phoneConnection: null,
internet: null,
videoSurveillance: null,
cableTV: null
};
for (const infrastructureId of infrastructureIds) {
switch (infrastructureId) {
case 1:
result.electricity = true;
break;
case 2:
result.water = true;
break;
case 4:
result.gas = true;
break;
case 5:
result.drainageSystem = true;
break;
case 7:
case 8:
result.phoneConnection = true;
break;
case 10:
result.internet = true;
break;
case 11:
result.cableTV = true;
break;
case 16:
case 17:
result.videoSurveillance = true;
break;
}
}
return result;
}
getFloorNumberFromFloorId(floorsIdText) {
// floorIdText can be array of numbers, separated by comma or number
// just extracting floor number from first element
const floorsId = floorsIdText.split(",");
if (floorsId.length === 0) {
return null;
}
const firstFloorId = parseInt(floorsId[0]);
// 1 pod
// 2 sut
// 3 raz
// 4 pri
// 5 vpri
// 6 prv
// 7 dru
// 8 tre
// 9 čet
// 10 man
// 11
// 12 pot
// 13 vpot
// 14 tav
// 15 pet
const floorNumber = [
-1,
-1,
0,
0,
1,
1,
2,
3,
4,
null,
null,
null,
null,
null,
5
];
return floorNumber[firstFloorId - 1] || null;
}
getNumberOfFloorsFromFloorId(floorsIdText) {
// floorIdText can be array of numbers, separated by comma or number
const floorIds = floorsIdText.split(",");
if (floorIds.length === 0) {
return null;
}
let noOfFloors = floorIds.length;
// Floors of 'suteren', 'podrum', 'tavan' and 'potkrovlje' are not counted
floorIds.forEach(id => {
if (
parseInt(id) === 1 ||
parseInt(id) === 2 ||
parseInt(id) === 12 ||
parseInt(id) === 14
) {
noOfFloors--;
}
});
return noOfFloors;
}
checkBasemAtticFromFloors(floorsIdText) {
// floorIdText can be array of numbers, separated by comma or number
const floorIds = floorsIdText.split(",");
let check = false;
if (floorIds.length === 0) {
check = false;
}
//If floors 'suteren', 'podrum', 'tavan' and 'potkrovlje' exists then tag for basement-attic is true
floorIds.forEach(id => {
if (
parseInt(id) === 1 ||
parseInt(id) === 2 ||
parseInt(id) === 12 ||
parseInt(id) === 14
) {
check = true;
}
});
return check;
}
async sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async saveCrawledResults(results) {
const savers = this.savers;
// for (const saver of savers) {
// await saver.save(results);
// }
//For now, we use only Postgres saver, so ...
return savers[0].save(results);
//so that we can use some sequelize options and information when data is inserted
}
}
module.exports = RentalCrawler;

View File

@@ -1,611 +0,0 @@
"use strict";
const fetch = require("node-fetch");
const cheerio = require("cheerio");
const moment = require("moment-timezone");
const {
AD_TYPE,
AD_CATEGORY,
AD_AGENCY,
AD_STATUS,
CRAWLER_AD_TYPE,
FURNISHING_TYPE,
HEATING_TYPE
} = require("../../common/enums");
const {
PRINT_CRAWLER_DEBUG,
DEFAULT_TIMEZONE
} = require("../../config/appConfig");
const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
const SALJIC_ENUMS = {
SALJIC_AD_TYPE: {
[CRAWLER_AD_TYPE.ALL]: "&input_vrsta=",
[CRAWLER_AD_TYPE.ONLY_SELL]: "&input_vrsta=1",
[CRAWLER_AD_TYPE.ONLY_RENT]: "&input_vrsta=2"
},
SALJIC_AD_CATEGORY: {
[AD_CATEGORY.ALL.id]: "&input_kategorija=",
[AD_CATEGORY.FLAT.id]: "&input_kategorija=15",
[AD_CATEGORY.HOUSE.id]: "&input_kategorija=9",
[AD_CATEGORY.LAND.id]: "&input_kategorija=5", //3 and 4 also gradjevinsko
[AD_CATEGORY.OFFICE.id]: "&input_kategorija=8",
[AD_CATEGORY.APARTMENT.id]: "&input_kategorija=1",
[AD_CATEGORY.GARAGE.id]: "&input_kategorija=2"
//[AD_CATEGORY.COTTAGE.id]: ""
}
};
class SaljicCrawler {
constructor(
savers = [],
crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
maxPages = 5000,
maxResultsPerPage = 5000,
ignoredUsernames = [],
delayBetweenPages = 1000
) {
this.savers = savers;
this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
this.crawlerAdTypes = crawlerAdTypes;
this.crawlerAdCategories = crawlerAdCategories;
this.maxResultsPerPage = maxResultsPerPage;
this.delayBetweenPages = delayBetweenPages;
}
async crawl() {
const crawlAdCategories = this.crawlerAdCategories;
const newRealEstates = [];
if (crawlAdCategories) {
const indexGenerators = [];
for (const adCategory of crawlAdCategories) {
indexGenerators.push(this.categoryIndexer(adCategory));
}
//
//console.log(indexGenerators);
//
let done = false;
while (!done) {
const categoryIndexerPromises = [];
const generatorsToRemove = [];
for (const indexGenerator of indexGenerators) {
categoryIndexerPromises.push(indexGenerator.next());
generatorsToRemove.push(false);
}
const singlePageResults = await Promise.all(categoryIndexerPromises);
const entries = singlePageResults.entries();
for (const [index, { value: singlePageResult }] of entries) {
if (singlePageResult) {
const saveResults = await this.saveCrawledResults(singlePageResult);
const { newRecords } = saveResults;
newRealEstates.push(...newRecords);
if (
Array.isArray(newRecords) &&
newRecords.length === 0 &&
!SALJIC_FORCE_CRAWL
) {
generatorsToRemove[index] = true;
}
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;
// console.log("Generator ", index + 1, "has no more pages");
}
}
// console.log("Generators state : ", generatorsToRemove);
for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
if (generatorsToRemove[i]) {
// console.log("\tRemove generator ", i + 1);
indexGenerators.splice(i, 1);
}
}
if (indexGenerators.length === 0) {
done = true;
}
await this.sleep(this.delayBetweenPages);
}
}
return newRealEstates;
}
async *categoryIndexer(adCategory) {
let pageToIndex = 1;
const urlAdTypePart = SALJIC_ENUMS.SALJIC_AD_TYPE[this.crawlerAdTypes];
const urlCategoryPart = SALJIC_ENUMS.SALJIC_AD_CATEGORY[adCategory];
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
while (true) {
const urlPagePart = pageToIndex === 1 ? "" : (pageToIndex - 1) * 2 * 11;
const urlPageToCrawl = `${this.baseUrl}?order_by=${urlAdTypePart}${urlCategoryPart}&per_page=${urlPagePart}`;
const singlePageResults = await this.indexSinglePage(
urlPageToCrawl,
this.maxResultsPerPage
);
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
yield singlePageResults;
} else {
return undefined;
}
++pageToIndex;
if (pageToIndex === this.maxPages) {
return undefined;
}
}
} else {
return undefined;
}
}
async indexSinglePage(url, maxResultsPerPage) {
if (PRINT_CRAWLER_DEBUG) {
console.log("[SALJIC] Index page : ", url);
}
try {
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
let hrefs = [];
$("#shop")
.find(".product")
.each((i, elem) => {
const href = $(elem)
.find("a")
.first()
.attr("href");
if (href) {
hrefs.push(href);
}
});
let adTypesTmp = [];
$("#shop")
.find(".product")
.each((i, elem) => {
const adType = $(elem)
.find(".trakica-search-page")
.text()
.trim();
if (adType) {
adTypesTmp.push(adType);
}
});
//Converting to AD_TYPE
const adTypes = adTypesTmp.map(adTypeText => {
return this.getAdTypeId(adTypeText);
});
//Converting to absolute URLs
const hrefsAbs = hrefs.map(link => {
return "https://www.saljicnekretnine.ba" + link;
});
let actualNoOfResults =
hrefsAbs.length <= maxResultsPerPage
? hrefsAbs.length
: maxResultsPerPage;
const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
}
const scrapedData = await Promise.all(asyncScraping);
const filteredScrapedData = scrapedData.filter(adData => !!adData);
return filteredScrapedData;
} catch (e) {
console.error("[SALJIC] Exception caught:" + e);
return [];
}
}
async scrapeAd(url, adType) {
console.log("[SALJIC] Scraping : ", url);
try {
const adPageSource = await fetch(url);
const body = await adPageSource.text();
const $ = cheerio.load(body);
// No information for status ex. PRODAN
const status = AD_STATUS.STATUS_NORMAL;
//Extracting agency ID from url
const agencyObjectId = parseInt(url.substring(46, url.length));
//Extracting main properties
const propertySelectors = {
title:
"div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
price:
"div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
streetName:
"div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
descriptions:
"div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
latAndLong:
"div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
};
const title = $(propertySelectors.title)
.text()
.replace(/(\r\n|\n|\r)/gm, "")
.replace(/ {1,}/g, " ")
.trim();
const priceText = $(propertySelectors.price)
.text()
.replace(/(\r\n|\n|\r)/gm, "")
.replace(/ {1,}/g, " ")
.trim();
const price =
priceText === "CIJENA NA UPIT"
? null
: parseFloat(
priceText.substring(8, priceText.length - 3).replace(",", "")
);
const streetName = $(propertySelectors.streetName)
.text()
.replace(/(\r\n|\n|\r)/gm, "")
.trim();
const descriptions = $(propertySelectors.descriptions)
.text()
.trim();
const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
const latText = latAndLongSrc.substring(
latAndLongSrc.indexOf("marker=") + 7,
latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
);
const longText = latAndLongSrc.substring(
latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
latAndLongSrc.length
);
const locationLat = parseFloat(latText) || null;
const locationLong = parseFloat(longText) || null;
//====== DETAIL INFORMATION FIELDS ==========
let area,
gardenSize,
numberOfRooms = null,
numberOfFloors = null,
floor = null,
accessRoadType = null,
heatingType = null,
furnishingType = null,
balcony = null,
newBuilding = null,
elevator = null,
water = null,
electricity = null,
drainageSystem = null,
registeredInZkBooks = null,
recentlyAdapted = null,
parking = null,
garage = null,
gas = null,
antiTheftDoor = null,
airCondition = null,
phoneConnection = null,
cableTV = null,
internet = null,
basementAttic = null,
storeRoom = null,
videoSurveillance = null,
alarm = null,
suitableForStudents = null,
includingBills = null,
animalsAllowed = null,
pool = null,
urbanPlanPermit = null,
buildingPermit = null,
utilityConnection = null,
distanceToRiver = null;
let publishedDate = null;
let renewedDate = null;
let realEstateType;
let numberOfViewsAgency = null;
//Extracting data - Glavne karakteristike
let mainFieldIndex = 1;
do {
const mainFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.list-group-item:nth-child(${mainFieldIndex})`;
const mainField = $(mainFieldSelector)
.text()
.replace(/[\n\r\t]/gm, "")
.trim();
const mainFieldTitle = mainField.substring(0, mainField.indexOf(" "));
const mainFieldValue = mainField
.substring(mainField.indexOf(" "), mainField.length)
.trim();
switch (mainFieldTitle) {
case "Površina":
area = parseFloat(
mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
);
break;
case "Okućnica":
gardenSize = parseFloat(
mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
);
break;
case "Broj soba":
numberOfRooms = parseInt(mainFieldValue);
break;
case "Broj spratova":
numberOfFloors = parseInt(mainFieldValue);
break;
case "Sprat":
floor = parseInt(mainFieldValue);
break;
case "Godina renoviranja":
recentlyAdapted = true;
break;
case "Broj parking mjesta":
parking = true;
break;
case "Dostupno od":
const day = mainFieldValue.substring(0, 2);
const month = mainFieldValue.substring(3, 5);
const year = mainFieldValue.substring(6, mainFieldValue.length);
publishedDate = new Date(`${month}/${day}/${year}`);
break;
default:
break;
}
if (mainFieldTitle === "") {
break;
}
mainFieldIndex++;
} while (true);
//Extracting data - Sadrzaji
let additionalFieldIndex = 1;
do {
const additionalFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.border-color.col-md-5.col-md-offset-1.col-md-pull-1.list-group-item-bottom:nth-child(${additionalFieldIndex})`;
const additionalField = $(additionalFieldSelector)
.text()
.trim();
if (additionalFieldIndex === 1) {
//Extracting data of real estate type
const categoryTmp = additionalField
.replace(/[\n\r\t]/gm, "")
.substring(
additionalField.indexOf("Kategorija") + 10,
additionalField.length
)
.trim();
realEstateType = this.getAdCategoryId(categoryTmp);
} else {
switch (additionalField) {
case "Internet":
internet = true;
break;
case "Garaža":
garage = true;
break;
case "Klima":
airCondition = true;
break;
case "Balkon":
balcony = true;
break;
case "Ostava":
storeRoom = true;
break;
case "Podrum":
basementAttic = true;
break;
case "Blindirana vrata":
antiTheftDoor = true;
break;
case "Voda":
water = true;
break;
case "Kablovska":
cableTV = true;
break;
case "Uknjiženo":
registeredInZkBooks = true;
break;
case "Grijanje - centralno":
heatingType = HEATING_TYPE.CENTRAL_CITY.id;
break;
case "Grijanje - plin":
heatingType = HEATING_TYPE.GAS.id;
break;
case "Grijanje - struja":
heatingType = HEATING_TYPE.ELECTRICITY.id;
break;
case "Grijanje":
heatingType = HEATING_TYPE.OTHER.id;
break;
case "Plin":
gas = true;
break;
case "Namješten":
furnishingType = FURNISHING_TYPE.FURNISHED.id;
break;
case "Alarm":
alarm = true;
break;
case "Video nadzor":
videoSurveillance = true;
break;
case "Lift":
elevator = true;
break;
case "Novogradnja":
newBuilding = true;
break;
default:
break;
}
}
if (additionalField === "") {
break;
}
additionalFieldIndex++;
} while (true);
//If no published date it takes current date of crawling
if (publishedDate) {
renewedDate = new Date();
} else {
publishedDate = new Date();
renewedDate = new Date();
}
const data = {
url,
agencyObjectId,
originAgencyName: AD_AGENCY.SALJIC,
realEstateType,
adType,
title,
price,
area,
gardenSize,
shortDescription: descriptions.substring(0, descriptions.indexOf(".")),
longDescription: descriptions,
streetNumber: 0,
streetName,
locality: "",
municipality: "",
city: "",
region: "",
entity: "",
country: "",
locationLat,
locationLong,
adStatus: status,
publishedDate,
renewedDate,
numberOfRooms,
numberOfFloors,
floor,
accessRoadType,
heatingType,
furnishingType,
balcony,
newBuilding,
elevator,
water,
electricity,
drainageSystem,
registeredInZkBooks,
recentlyAdapted,
parking,
garage,
gas,
antiTheftDoor,
airCondition,
phoneConnection,
cableTV,
internet,
basementAttic,
storeRoom,
videoSurveillance,
alarm,
suitableForStudents,
includingBills,
animalsAllowed,
pool,
urbanPlanPermit,
buildingPermit,
utilityConnection,
distanceToRiver,
numberOfViewsAgency
};
console.log(data);
return data;
} catch (e) {
console.error("Exception caught: " + e.message, "\r\nURL:", url);
}
return null;
}
//======= HELPER FUNCTIONS =============
getAdCategoryId(categoryText) {
switch (categoryText) {
case "Stan":
return AD_CATEGORY.FLAT.id;
case "Građevinsko zemljiste":
return AD_CATEGORY.LAND.id;
case "Industrijsko zemljiste":
return AD_CATEGORY.LAND.id;
case "Poljoprivredno zemljiste":
return AD_CATEGORY.LAND.id;
case "Kuća":
return AD_CATEGORY.HOUSE.id;
case "Poslovni prostor":
return AD_CATEGORY.OFFICE.id;
case "Kancelarije":
return AD_CATEGORY.OFFICE.id;
case "Apartmani":
return AD_CATEGORY.APARTMENT.id;
case "Garaža":
return AD_CATEGORY.GARAGE.id;
case "Vikendica":
return AD_CATEGORY.COTTAGE.id;
default:
return undefined;
}
}
getAdTypeId(adTypeText) {
switch (adTypeText) {
case "PRODAJA":
return AD_TYPE.AD_TYPE_SALE.stringId;
case "NAJAM":
return AD_TYPE.AD_TYPE_RENT.stringId;
default:
return undefined;
}
}
async sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async saveCrawledResults(results) {
const savers = this.savers;
// for (const saver of savers) {
// await saver.save(results);
// }
//For now, we use only Postgres saver, so ...
return savers[0].save(results);
//so that we can use some sequelize options and information when data is inserted
}
}
module.exports = SaljicCrawler;

View File

@@ -1,931 +0,0 @@
const regions = [
{
name: " Sarajevo",
id: "sarajevo",
olxid: "9",
municipalities: [
{
name: "Hadžići",
id: "hadii",
olxid: "3817"
},
{
name: "Ilidža",
id: "ilida",
olxid: "3879"
},
{
name: "Ilijaš",
id: "ilija",
olxid: "3892"
},
{
name: "Sarajevo - Centar",
id: "sarajevocentar",
olxid: "3812"
},
{
name: "Sarajevo-Novi Grad",
id: "sarajevonovigrad",
olxid: "3969"
},
{
name: "Sarajevo-Novo Sarajevo",
id: "sarajevonovosarajevo",
olxid: "5896"
},
{
name: "Sarajevo-Stari Grad",
id: "sarajevostarigrad",
olxid: "4048"
},
{
name: "Trnovo",
id: "trnovo",
olxid: "4063"
},
{
name: "Vogošća",
id: "vogoa",
olxid: "4126"
}
]
},
{
name: " Unsko-sanski",
id: "unskosanski",
olxid: "1",
municipalities: [
{
name: "Bihać",
id: "biha",
olxid: "75"
},
{
name: "Bosanska Krupa",
id: "bosanskakrupa",
olxid: "373"
},
{
name: "Bosanski Petrovac",
id: "bosanskipetrovac",
olxid: "504"
},
{
name: "Bužim",
id: "buim",
olxid: "374"
},
{
name: "Cazin",
id: "cazin",
olxid: "857"
},
{
name: "Ključ",
id: "klju",
olxid: "2362"
},
{
name: "Sanski Most",
id: "sanskimost",
olxid: "3738"
},
{
name: "Velika Kladuša",
id: "velikakladua",
olxid: "5122"
}
]
},
{
name: " Posavski",
id: "posavski",
olxid: "2",
municipalities: [
{
name: "Domaljevac",
id: "domaljevac",
olxid: "6144"
},
{
name: "Odžak",
id: "odak",
olxid: "424"
},
{
name: "Orašje",
id: "oraje",
olxid: "3252"
},
{
name: "Šamac",
id: "amac",
olxid: "540"
}
]
},
{
name: " Tuzlanski",
id: "tuzlanski",
olxid: "3",
municipalities: [
{
name: "Banovići",
id: "banovii",
olxid: "2"
},
{
name: "Doboj-Istok",
id: "dobojistok",
olxid: "1090"
},
{
name: "Gradačac",
id: "gradaac",
olxid: "1854"
},
{
name: "Gračanica",
id: "graanica",
olxid: "1826"
},
{
name: "Kalesija",
id: "kalesija",
olxid: "2129"
},
{
name: "Kladanj",
id: "kladanj",
olxid: "2319"
},
{
name: "Lukavac",
id: "lukavac",
olxid: "2840"
},
{
name: "Sapna",
id: "sapna",
olxid: "5699"
},
{
name: "Srebrenik",
id: "srebrenik",
olxid: "4391"
},
{
name: "Teočak",
id: "teoak",
olxid: "5010"
},
{
name: "Tuzla",
id: "tuzla",
olxid: "4944"
},
{
name: "Čelić",
id: "eli",
olxid: "2801"
},
{
name: "Živinice",
id: "ivinice",
olxid: "5774"
}
]
},
{
name: " Zeničko-dobojski",
id: "zenickodobojski",
olxid: "4",
municipalities: [
{
name: "Breza",
id: "breza",
olxid: "704"
},
{
name: "Doboj-Jug",
id: "dobojjug",
olxid: "1122"
},
{
name: "Kakanj",
id: "kakanj",
olxid: "2022"
},
{
name: "Maglaj",
id: "maglaj",
olxid: "2941"
},
{
name: "Olovo",
id: "olovo",
olxid: "1925"
},
{
name: "Tešanj",
id: "teanj",
olxid: "4594"
},
{
name: "Usora",
id: "usora",
olxid: "1087"
},
{
name: "Vareš",
id: "vare",
olxid: "5037"
},
{
name: "Visoko",
id: "visoko",
olxid: "5171"
},
{
name: "Zavidovići",
id: "zavidovii",
olxid: "5548"
},
{
name: "Zenica",
id: "zenica",
olxid: "4571"
},
{
name: "Žepče",
id: "epe",
olxid: "2940"
}
]
},
{
name: " Bosansko-podrinjski",
id: "bosanskopodrinjski",
olxid: "5",
municipalities: [
{
name: "Foča",
id: "foa",
olxid: "1289"
},
{
name: "Goražde",
id: "gorade",
olxid: "1588"
},
{
name: "Pale",
id: "pale",
olxid: "3546"
}
]
},
{
name: " Srednjobosanski",
id: "srednjobosanski",
olxid: "6",
municipalities: [
{
name: "Bugojno",
id: "bugojno",
olxid: "732"
},
{
name: "Busovača",
id: "busovaa",
olxid: "810"
},
{
name: "Dobretići",
id: "dobretii",
olxid: "4151"
},
{
name: "Donji Vakuf",
id: "donjivakuf",
olxid: "1160"
},
{
name: "Fojnica",
id: "fojnica",
olxid: "1407"
},
{
name: "Gornji Vakuf - Uskoplje",
id: "gornjivakufuskoplje",
olxid: "1775"
},
{
name: "Jajce",
id: "jajce",
olxid: "1960"
},
{
name: "Kiseljak",
id: "kiseljak",
olxid: "2237"
},
{
name: "Kreševo",
id: "kreevo",
olxid: "2608"
},
{
name: "Novi Travnik",
id: "novitravnik",
olxid: "3477"
},
{
name: "Travnik",
id: "travnik",
olxid: "4678"
},
{
name: "Vitez",
id: "vitez",
olxid: "5422"
}
]
},
{
name: " Hercegovačko-neretvanski",
id: "hercegovackoneretvanski",
olxid: "7",
municipalities: [
{
name: "Grad Mostar",
id: "gradmostar",
olxid: "3017"
},
{
name: "Jablanica",
id: "jablanica",
olxid: "1930"
},
{
name: "Konjic",
id: "konjic",
olxid: "2169"
},
{
name: "Neum",
id: "neum",
olxid: "3111"
},
{
name: "Prozor",
id: "prozor",
olxid: "3421"
},
{
name: "Ravno",
id: "ravno",
olxid: "4769"
},
{
name: "Stolac",
id: "stolac",
olxid: "4439"
},
{
name: "Čapljina",
id: "apljina",
olxid: "947"
},
{
name: "Čitluk",
id: "itluk",
olxid: "1009"
}
]
},
{
name: " Zapadno-hercegovački",
id: "zapadnohercegovacki",
olxid: "8",
municipalities: [
{
name: "Grude",
id: "grude",
olxid: "1892"
},
{
name: "Ljubuški",
id: "ljubuki",
olxid: "2905"
},
{
name: "Posušje",
id: "posuje",
olxid: "3268"
},
{
name: "Široki Brijeg",
id: "irokibrijeg",
olxid: "2708"
}
]
},
{
name: " Livanjski",
id: "livanjski",
olxid: "10",
municipalities: [
{
name: "Bosansko Grahovo",
id: "bosanskograhovo",
olxid: "560"
},
{
name: "Drvar",
id: "drvar",
olxid: "4640"
},
{
name: "Glamoč",
id: "glamo",
olxid: "1533"
},
{
name: "Kupres",
id: "kupres",
olxid: "2635"
},
{
name: "Livno",
id: "livno",
olxid: "2741"
},
{
name: "Tomislavgrad",
id: "tomislavgrad",
olxid: "1228"
}
]
},
{
name: " Banjalučka",
id: "banjalučka",
olxid: "14",
municipalities: [
{
name: "Banja Luka",
id: "banjaluka",
olxid: "21"
},
{
name: "Gradiška",
id: "gradika",
olxid: "305"
},
{
name: "Istočni Drvar",
id: "istonidrvar",
olxid: "4662"
},
{
name: "Jezero",
id: "jezero",
olxid: "1965"
},
{
name: "Kneževo",
id: "kneevo",
olxid: "4147"
},
{
name: "Kostajnica",
id: "kostajnica",
olxid: "6142"
},
{
name: "Kotor Varoš",
id: "kotorvaro",
olxid: "2574"
},
{
name: "Kozarska Dubica",
id: "kozarskadubica",
olxid: "244"
},
{
name: "Krupa na uni",
id: "krupanauni",
olxid: "382"
},
{
name: "Kupres ",
id: "kupres",
olxid: "2654"
},
{
name: "Laktaši",
id: "laktai",
olxid: "2671"
},
{
name: "Mrkonjić Grad",
id: "mrkonjigrad",
olxid: "3073"
},
{
name: "Novi Grad",
id: "novigrad",
olxid: "444"
},
{
name: "Oštra Luka",
id: "otraluka",
olxid: "3737"
},
{
name: "Petrovac",
id: "petrovac",
olxid: "515"
},
{
name: "Prijedor",
id: "prijedor",
olxid: "3287"
},
{
name: "Prnjavor",
id: "prnjavor",
olxid: "3358"
},
{
name: "Ribnik",
id: "ribnik",
olxid: "2365"
},
{
name: "Srbac",
id: "srbac",
olxid: "4271"
},
{
name: "Čelinac",
id: "elinac",
olxid: "979"
},
{
name: "Šipovo",
id: "ipovo",
olxid: "4509"
}
]
},
{
name: " Dobojsko-Bijeljinska",
id: "dobojskobijeljinska",
olxid: "15",
municipalities: [
{
name: "Bijeljina",
id: "bijeljina",
olxid: "123"
},
{
name: "Bosanski Brod",
id: "bosanskibrod",
olxid: "421"
},
{
name: "Derventa",
id: "derventa",
olxid: "1030"
},
{
name: "Doboj",
id: "doboj",
olxid: "1088"
},
{
name: "Donji Žabar",
id: "donjiabar",
olxid: "3254"
},
{
name: "Lopare",
id: "lopare",
olxid: "2800"
},
{
name: "Lukavac",
id: "lukavac",
olxid: "6029"
},
{
name: "Modriča",
id: "modria",
olxid: "2996"
},
{
name: "Pelagićevo",
id: "pelagievo",
olxid: "1856"
},
{
name: "Petrovo",
id: "petrovo",
olxid: "1827"
},
{
name: "Stanari",
id: "stanari",
olxid: "1148"
},
{
name: "Teslić",
id: "tesli",
olxid: "4549"
},
{
name: "Tešanj",
id: "teanj",
olxid: "4636"
},
{
name: "Travnik",
id: "travnik",
olxid: "4692"
},
{
name: "Tuzla",
id: "tuzla",
olxid: "4966"
},
{
name: "Ugljevik",
id: "ugljevik",
olxid: "5009"
},
{
name: "Vukosavlje",
id: "vukosavlje",
olxid: "3197"
},
{
name: "Šamac",
id: "amac",
olxid: "539"
}
]
},
{
name: " Sarajevsko-Zvornička",
id: "sarajevskozvornicka",
olxid: "16",
municipalities: [
{
name: "Bratunac",
id: "bratunac",
olxid: "595"
},
{
name: "Han Pijesak",
id: "hanpijesak",
olxid: "1904"
},
{
name: "Ilijaš",
id: "ilija",
olxid: "3947"
},
{
name: "Istočni Stari Grad",
id: "istonistarigrad",
olxid: "4049"
},
{
name: "Kasindo",
id: "kasindo",
olxid: "3880"
},
{
name: "Kladanj",
id: "kladanj",
olxid: "2325"
},
{
name: "Lukavica",
id: "lukavica",
olxid: "3971"
},
{
name: "Milići",
id: "milii",
olxid: "6143"
},
{
name: "Olovo",
id: "olovo",
olxid: "3221"
},
{
name: "Osmaci",
id: "osmaci",
olxid: "2128"
},
{
name: "Pale",
id: "pale",
olxid: "3978"
},
{
name: "Rogatica",
id: "rogatica",
olxid: "3529"
},
{
name: "Rudo",
id: "rudo",
olxid: "3648"
},
{
name: "Sarajevo-Novi Grad",
id: "sarajevonovigrad",
olxid: "6069"
},
{
name: "Sokolac",
id: "sokolac",
olxid: "4183"
},
{
name: "Srebrenica",
id: "srebrenica",
olxid: "4310"
},
{
name: "Trnovo",
id: "trnovo",
olxid: "4067"
},
{
name: "Ustiprača",
id: "ustipraa",
olxid: "1593"
},
{
name: "Višegrad",
id: "viegrad",
olxid: "5259"
},
{
name: "Vlasenica",
id: "vlasenica",
olxid: "5456"
},
{
name: "Zvornik",
id: "zvornik",
olxid: "5684"
},
{
name: "Šekovići",
id: "ekovii",
olxid: "4475"
},
{
name: "Žepa",
id: "epa",
olxid: "1906"
}
]
},
{
name: " Trebinjsko-Fočanska",
id: "trebinjskofocanska",
olxid: "17",
municipalities: [
{
name: "Berkovići",
id: "berkovii",
olxid: "4441"
},
{
name: "Bileća",
id: "bilea",
olxid: "183"
},
{
name: "Foča",
id: "foa",
olxid: "1287"
},
{
name: "Gacko",
id: "gacko",
olxid: "1462"
},
{
name: "Istočni Mostar",
id: "istonimostar",
olxid: "3038"
},
{
name: "Kalinovik",
id: "kalinovik",
olxid: "2164"
},
{
name: "Ljubinje",
id: "ljubinje",
olxid: "2884"
},
{
name: "Nevesinje",
id: "nevesinje",
olxid: "3138"
},
{
name: "Trebinje",
id: "trebinje",
olxid: "4766"
},
{
name: "Čajniče",
id: "ajnie",
olxid: "911"
}
]
},
{
name: "Distrikt Brčko",
id: "distriktbrcko",
olxid: "12",
municipalities: [
{
name: "Brčko",
id: "brko",
olxid: "645"
}
]
}
];
const getRegions = () => {
return regions.map(g => ({ name: g.name, id: g.id, olxid: g.olxid }));
};
const getRegion = regionId => {
return regions.find(region => region.id === regionId);
};
const getRegionName = regionId => {
const region = getRegion(regionId);
return region && region.name ? region.name : null;
};
const getMunicipalitiesForRegion = regionId => {
const region = getRegion(regionId);
return region && region.municipalities ? region.municipalities : null;
};
const getMunicipality = (regionId, municipalityId) => {
const region = getRegion(regionId);
if (!region) {
return null;
}
const municipality = region.municipalities.find(
municipality => municipality.id === municipalityId
);
if (!municipality) {
return null;
}
return municipality;
};
const getMunicipalityName = (regionId, municipalityId) => {
const region = getRegion(regionId);
if (!region) {
return null;
}
const municipality = region.municipalities.find(
municipality => municipality.id === municipalityId
);
if (!municipality) {
return null;
}
return municipality.name;
};
module.exports = {
getRegion,
getRegions,
getRegionName,
getMunicipalitiesForRegion,
getMunicipalityName,
getMunicipality
};

View File

@@ -1,304 +0,0 @@
"use strict";
const db = require("../../models/index");
const sequelize = require("sequelize");
const Op = sequelize.Op;
const bulkUpsertRealEstates = async realEstateData => {
try {
const fieldsToUpdateIfDuplicate = [
"realEstateType",
"adType",
"price",
"area",
"streetNumber",
"streetName",
"locality",
"municipality",
"city",
"region",
"entity",
"country",
"locationLat",
"locationLong",
"title",
"shortDescription",
"longDescription",
"gardenSize",
"adStatus",
"updatedAt",
"renewedDate",
"numberOfRooms",
"numberOfFloors",
"floor",
"accessRoadType",
"heatingType",
"furnishingType",
"balcony",
"newBuilding",
"elevator",
"water",
"electricity",
"drainageSystem",
"registeredInZkBooks",
"recentlyAdapted",
"parking",
"garage",
"gas",
"antiTheftDoor",
"airCondition",
"phoneConnection",
"cableTV",
"internet",
"basementAttic",
"storeRoom",
"videoSurveillance",
"alarm",
"suitableForStudents",
"includingBills",
"animalsAllowed",
"pool",
"urbanPlanPermit",
"buildingPermit",
"utilityConnection",
"distanceToRiver",
"numberOfViewsAgency"
];
const order = [["updatedAt", "desc"]];
return await db.RealEstate.bulkCreate(realEstateData, {
updateOnDuplicate: fieldsToUpdateIfDuplicate,
returning: true,
order
});
} catch (e) {
console.log("Error bulk upserting realEstates : ", e);
}
};
const getRealEstateById = async id => {
return db.RealEstate.findByPk(id);
};
const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
const {
priceMin,
priceMax,
sizeMin,
sizeMax,
adType,
realEstateType,
areaToSearch,
gardenSizeMin,
gardenSizeMax,
numberOfRoomsMin,
numberOfRoomsMax,
numberOfFloorsMin,
numberOfFloorsMax,
floorMin,
floorMax,
includeIncompleteAds,
balcony,
elevator,
newBuilding,
accessRoadType
} = searchRequest;
const longitudeColumn = sequelize.col("locationLong");
const latitudeColumn = sequelize.col("locationLat");
const pointGeometry = sequelize.fn(
"ST_Point",
longitudeColumn,
latitudeColumn
);
const pointWithSRID = sequelize.fn("ST_SetSRID", pointGeometry, 4326);
const areaToSearchAsGeometry = sequelize.fn(
"ST_GeomFromGeoJSON",
JSON.stringify(areaToSearch)
);
const areaToSearchWithSRID = sequelize.fn(
"ST_SetSRID",
areaToSearchAsGeometry,
4326
);
const contains = sequelize.fn(
"ST_Contains",
areaToSearchWithSRID,
pointWithSRID
);
const geoSearchQueryPart = sequelize.where(contains, true);
//General queries contain only attributes that are defined for every searchreq
//Query for case of complete ads
const query = {
adType,
realEstateType,
price: {
[Op.or]: {
[Op.and]: {
[Op.lte]: priceMax,
[Op.gte]: priceMin
},
[Op.is]: null
}
},
area: {
[Op.lte]: sizeMax,
[Op.gte]: sizeMin
},
[Op.and]: geoSearchQueryPart
};
//Query for case of incomplete ads
const queryIncludeIncomplete = {
adType,
realEstateType,
price: {
[Op.or]: {
[Op.and]: {
[Op.lte]: priceMax,
[Op.gte]: priceMin
},
[Op.is]: null
}
},
area: {
[Op.or]: {
[Op.and]: {
[Op.lte]: sizeMax,
[Op.gte]: sizeMin
},
[Op.is]: null
}
},
[Op.and]: geoSearchQueryPart
};
//Every other attribute is checked separately and included in query only if it is defined
if (gardenSizeMax && gardenSizeMin) {
query.gardenSize = {
[Op.lte]: gardenSizeMax,
[Op.gte]: gardenSizeMin
};
queryIncludeIncomplete.gardenSize = {
[Op.or]: {
[Op.and]: {
[Op.lte]: gardenSizeMax,
[Op.gte]: gardenSizeMin
},
[Op.is]: null
}
};
}
if (numberOfRoomsMin && numberOfRoomsMax) {
query.numberOfRooms = {
[Op.lte]: numberOfRoomsMax,
[Op.gte]: numberOfRoomsMin
};
queryIncludeIncomplete.numberOfRooms = {
[Op.or]: {
[Op.and]: {
[Op.lte]: numberOfRoomsMax,
[Op.gte]: numberOfRoomsMin
},
[Op.is]: null
}
};
}
if (numberOfFloorsMin && numberOfFloorsMax) {
query.numberOfFloors = {
[Op.lte]: numberOfFloorsMax,
[Op.gte]: numberOfFloorsMin
};
queryIncludeIncomplete.numberOfFloors = {
[Op.or]: {
[Op.and]: {
[Op.lte]: numberOfFloorsMax,
[Op.gte]: numberOfFloorsMin
},
[Op.is]: null
}
};
}
if (floorMin && floorMax) {
query.floor = {
[Op.lte]: floorMax,
[Op.gte]: floorMin
};
queryIncludeIncomplete.floor = {
[Op.or]: {
[Op.and]: {
[Op.lte]: floorMax,
[Op.gte]: floorMin
},
[Op.is]: null
}
};
}
if (balcony) {
query.balcony = {
[Op.eq]: balcony
};
queryIncludeIncomplete.balcony = {
[Op.or]: {
[Op.eq]: balcony,
[Op.is]: null
}
};
}
if (newBuilding) {
query.newBuilding = {
[Op.eq]: newBuilding
};
queryIncludeIncomplete.newBuilding = {
[Op.or]: {
[Op.eq]: newBuilding,
[Op.is]: null
}
};
}
if (elevator) {
query.elevator = {
[Op.eq]: elevator
};
queryIncludeIncomplete.elevator = {
[Op.or]: {
[Op.eq]: elevator,
[Op.is]: null
}
};
}
if (accessRoadType !== "ANY") {
query.accessRoadType = {
[Op.eq]: accessRoadType
};
queryIncludeIncomplete.accessRoadType = {
[Op.or]: {
[Op.eq]: accessRoadType,
[Op.is]: null
}
};
}
const order = [["updatedAt", "desc"]];
return db.RealEstate.findAll({
where: includeIncompleteAds ? queryIncludeIncomplete : query,
limit: maxResults,
order
});
};
module.exports = {
bulkUpsertRealEstates,
getRealEstateById,
findRealEstatesForSearchRequest
};

View File

@@ -1,180 +0,0 @@
"use strict";
const db = require("../../models/index");
const sequelize = require("sequelize");
const Op = sequelize.Op;
const { AD_CATEGORY } = require("../../common/enums");
const getSearchRequest = async searchRequestId => {
try {
return await db.SearchRequest.findByPk(searchRequestId);
} catch (error) {
console.log("searchrequest.js", error);
return null;
}
};
const createSearchRequest = async (searchRequestFields = {}) => {
return await db.SearchRequest.create(searchRequestFields);
};
const findSearchRequestsForRealEstate = async realEstate => {
const {
price,
area,
adType,
realEstateType,
locationLat,
locationLong,
accessRoadType,
balcony,
newBuilding,
elevator,
gardenSize,
numberOfRooms,
numberOfFloors,
floor
} = realEstate;
if (!locationLat || !locationLong) {
return [];
}
const stGeometry = sequelize.fn(
"ST_GEOMFROMTEXT",
`POINT (${locationLong} ${locationLat})`,
4326
);
const areaToSearchColumn = sequelize.col("areaToSearch");
const contains = sequelize.fn("ST_Contains", areaToSearchColumn, stGeometry);
const geoSearchQueryPart = sequelize.where(contains, true);
//General query contains only attributes that are defined for every RealEstate - not null
const query = {
adType,
realEstateType,
subscribed: true,
[Op.and]: geoSearchQueryPart
};
//Needed for defining which attribute should exist or not
const realEstateTypeObject = AD_CATEGORY[realEstateType];
//Needed to decide on including incomplete RealEstates data
let checkForIncompleteWanted = false;
//Attributes are checked separately and included in query only if defined
//Price and area should be defined for every property
if (price) {
query.priceMin = {
[Op.lte]: price
};
query.priceMax = {
[Op.gte]: price
};
}
if (area) {
query.sizeMin = {
[Op.lte]: area
};
query.sizeMax = {
[Op.gte]: area
};
} else {
checkForIncompleteWanted = true;
}
//Other attributes can be defined or not depending on RealEstate type
if (gardenSize) {
query.gardenSizeMin = {
[Op.lte]: gardenSize
};
query.gardenSizeMax = {
[Op.gte]: gardenSize
};
} else if (realEstateTypeObject.hasGardenSize) {
checkForIncompleteWanted = true;
}
if (numberOfRooms) {
query.numberOfRoomsMin = {
[Op.lte]: numberOfRooms
};
query.numberOfRoomsMax = {
[Op.gte]: numberOfRooms
};
} else if (realEstateTypeObject.hasNumberOfRoom) {
checkForIncompleteWanted = true;
}
if (numberOfFloors) {
query.numberOfFloorsMin = {
[Op.lte]: numberOfFloors
};
query.numberOfFloorsMax = {
[Op.gte]: numberOfFloors
};
} else if (realEstateTypeObject.hasNumberOfFloors) {
checkForIncompleteWanted = true;
}
if (floor) {
query.floorMin = {
[Op.lte]: floor
};
query.floorMax = {
[Op.gte]: floor
};
} else if (realEstateTypeObject.hasFloorProp) {
checkForIncompleteWanted = true;
}
if (accessRoadType) {
query.accessRoadType = {
[Op.or]: {
[Op.eq]: "ANY",
[Op.eq]: accessRoadType
}
};
} else if (realEstateTypeObject.hasAccesRoadType) {
checkForIncompleteWanted = true;
}
if (balcony) {
query.balcony = {
[Op.eq]: balcony
};
} else if (realEstateTypeObject.hasBalconyProp) {
checkForIncompleteWanted = true;
}
if (newBuilding) {
query.newBuilding = {
[Op.eq]: newBuilding
};
} else if (realEstateTypeObject.hasNewBuildingProp) {
checkForIncompleteWanted = true;
}
if (elevator) {
query.elevator = {
[Op.eq]: elevator
};
} else if (realEstateTypeObject.hasElevatorProp) {
checkForIncompleteWanted = true;
}
//If one of the attributes that exists for property type is null
//we include in query to check if incomplete real estates are accepted
if (checkForIncompleteWanted) {
query.includeIncompleteAds = {
[Op.eq]: true
};
}
return await db.SearchRequest.findAll({ where: query });
};
module.exports = {
getSearchRequest,
createSearchRequest,
findSearchRequestsForRealEstate
};

View File

@@ -1,54 +0,0 @@
"use strict";
const db = require("../../models/index");
const findRealEstatesForSearchRequest = async searchRequestId => {
const query = {
searchRequestId
};
const realEstatesModel = { model: db.RealEstate, as: "realEstates" };
const order = [[realEstatesModel, "updatedAt", "desc"]];
const include = [realEstatesModel];
const matches = await db.SearchRequestMatch.findAll({
where: query,
include,
order
});
const matchingRealEstates = [];
for (const match of matches) {
matchingRealEstates.push(...match.realEstates);
}
return matchingRealEstates;
};
const findNotNotifiedMatches = async () => {
const query = {
notified: false
};
const searchRequestsModel = { model: db.SearchRequest, as: "searchRequests" };
const realEstateModel = { model: db.RealEstate, as: "realEstates" };
const include = [searchRequestsModel, realEstateModel];
const matchingRecords = await db.SearchRequestMatch.findAll({
where: query,
include
});
return matchingRecords;
};
const addMatches = async matchingRecords => {
return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
ignoreDuplicates: true
});
};
module.exports = {
findRealEstatesForSearchRequest,
addMatches,
findNotNotifiedMatches
};

View File

@@ -1,8 +0,0 @@
const isValidEmail = email => {
const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
return email && email.length < 250 && simpleEmailRegex.test(email);
};
module.exports = {
isValidEmail
};

View File

@@ -1,127 +0,0 @@
"use strict";
const { MAX_REAL_ESTATES_IN_EMAIL, APP_URL } = require("../config/appConfig");
const { AD_CATEGORY } = require("../common/enums");
const generateEmailFooter = searchRequestId => {
return `<div>Ako želite prestati dobijati obavještenja za ovu pretragu, <a href="${APP_URL}/odjava/${searchRequestId}">odjavite ovdje</a></div>
<div>Ako želite pogledati ili promijeniti uslove za ovu pretragu, <a href="${APP_URL}/pregled/${searchRequestId}">pogledajte ovdje</a></div>
<br/>
<strong>Vaš,<br/>Kivi tim</strong>`;
};
const generateRealEstateLinks = realEstates => {
let realEstateLinks = "";
for (const realEstate of realEstates) {
const { id: realEstateId, title } = realEstate;
realEstateLinks += `<li><a href="${APP_URL}/redirect/${realEstateId}">${title}</a></li>`;
}
return realEstateLinks;
};
const generateNotificationEmail = (
realEstates,
searchRequestId,
dailyNotification = false
) => {
const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
const realEstatesToShow = truncateList
? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
: realEstates;
const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
const emailFooter = generateEmailFooter(searchRequestId);
const asapMessageBody =
realEstates.length > 1
? "Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi"
: "Pronašli smo nekretninu koja odgovara Vašoj pretrazi";
const dailyMessageBody =
realEstates.length > 1
? "U posljednja 24h objavljene su sljedeće nekretnine koje odgovaraju uslovima Vaše pretrage"
: "U posljednja 24h objavljena je sljedeća nekretnina koja odgovara uslovima Vaše pretrage";
const messageBody = dailyNotification ? dailyMessageBody : asapMessageBody;
return `<h3>Zdravo</h3>
<h4>${messageBody}</h4>
<div>
${realEstateLinks}
<div/>
${moreRealEstates}
</div>
<br/>
${emailFooter}`;
};
const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
const {
id,
gardenSizeMin,
gardenSizeMax,
sizeMin,
sizeMax,
priceMin,
priceMax
} = searchRequest;
const realEstateLinks = generateRealEstateLinks(matchingRealEstates);
const instantRealEstatesText = `<br/>
<div>
U međuvremenu pogledajte neke od nedavno objavljenih nekretnina koje odgovaraju Vašim uslovima pretrage :<br/>
${realEstateLinks}
</div>`;
const gardenSize = realEstateType.hasGardenSize
? `<div><strong>Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2</strong></div>`
: ``;
const emailFooter = generateEmailFooter(id);
return `<h3>Zdravo</h3>
<div>Naručili ste da Vam javimo ako se nekretnina sa navedenim uslovima pojavi u oglasima:</div>
<br/>
<div>
<div><strong>Tip nekretnine: </strong>${realEstateType.title}</div>
<div><strong>Kvadratura nekretnine:</strong> Od ${sizeMin} do ${sizeMax} m2</div>
${gardenSize}
<div><strong>Cijena:</strong> ${priceMin} do ${priceMax} KM</div>
</div>
${matchingRealEstates.length > 0 ? instantRealEstatesText : ""}
<br/>
${emailFooter}`;
};
const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => {
if (numberOfRealEstates === 1) {
return `Kivi: ${singleRealEstateTitle}`;
}
const leastSignificantDigit = numberOfRealEstates % 10;
const numberWithoutLastDigit = Math.floor(numberOfRealEstates / 10);
const secondLeastSignificantDigit = numberWithoutLastDigit % 10;
if (leastSignificantDigit === 1 && secondLeastSignificantDigit !== 1) {
return `Kivi : ${numberOfRealEstates} nova nekretnina`;
}
if (
leastSignificantDigit >= 2 &&
leastSignificantDigit <= 4 &&
secondLeastSignificantDigit !== 1
) {
return `Kivi: ${numberOfRealEstates} nove nekretnine`;
}
return `Kivi: ${numberOfRealEstates} novih nekretnina`;
};
module.exports = {
generateNotificationEmail,
generateNewSearchRequestEmail,
generateEmailSubject
};

View File

@@ -1,25 +0,0 @@
/**
* Force load with https on production environment
* https://devcenter.heroku.com/articles/http-routing#heroku-headers
*/
module.exports = function(environments, status) {
environments = environments || ["production"];
status = status || 301;
// console.log("New force SSL ");
// console.log("\tenvs : ", environments);
// console.log("\tstatus: ", status);
// console.log("\tENV : ", process.env.NODE_ENV);
return function(req, res, next) {
if (environments.indexOf(process.env.NODE_ENV) >= 0) {
if (req.headers["x-forwarded-proto"] !== "https") {
const urlToRedirectTo = `https://${req.hostname}${req.originalUrl}`;
// console.log("\tRedirect :", urlToRedirectTo);
res.redirect(status, urlToRedirectTo);
} else {
next();
}
} else {
next();
}
};
};

View File

@@ -1,12 +0,0 @@
const { getSearchRequest } = require("./db/searchRequest");
const currentSearchRequest = async req => {
const searchRequestId =
req && req.params ? req.params["searchRequestId"] : null;
if (!searchRequestId) return null;
return await getSearchRequest(searchRequestId);
};
module.exports = {
currentSearchRequest
};

View File

@@ -1,34 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.createTable("MarketAlerts", {
id: {
allowNull: false,
autoIncrement: true,
primaryKey: true,
type: Sequelize.INTEGER
},
olxUrl: {
type: Sequelize.STRING
},
lastDate: {
type: Sequelize.STRING
},
email: {
type: Sequelize.STRING,
allowNull: false
},
createdAt: {
allowNull: false,
type: Sequelize.DATE
},
updatedAt: {
allowNull: false,
type: Sequelize.DATE
}
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.dropTable("MarketAlerts");
}
};

View File

@@ -1,15 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn(
"RealEstateRequests",
"city",
Sequelize.STRING
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstateRequests", "city");
}
};

View File

@@ -1,15 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn(
"RealEstateRequests",
"place",
Sequelize.STRING
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstateRequests", "place");
}
};

View File

@@ -1,19 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.renameColumn(
"RealEstateRequests",
"place",
"municipality"
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.renameColumn(
"RealEstateRequests",
"municipality",
"place"
);
}
};

View File

@@ -1,11 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.renameColumn("RealEstateRequests", "city", "region");
},
down: (queryInterface, Sequelize) => {
return queryInterface.renameColumn("RealEstateRequests", "region", "city");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("RealEstateRequests", "size", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstateRequests", "size");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("RealEstateRequests", "gardenSize", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstateRequests", "gardenSize");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("RealEstateRequests", "price", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstateRequests", "price");
}
};

View File

@@ -1,19 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize
.query("CREATE EXTENSION postgis")
.then(([results, metadata]) => {
/// No result
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize
.query("DROP EXTENSION IF EXISTS postgis")
.then(([results, metadata]) => {
/// No result
});
}
};

View File

@@ -1,21 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize
.query(
'ALTER TABLE "RealEstateRequests" ADD COLUMN bounding_box geometry(Polygon);'
)
.then(([results, metadata]) => {
/// No result
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize
.query('ALTER TABLE "RealEstateRequests" DROP COLUMN bounding_box')
.then(([results, metadata]) => {
/// No result
});
}
};

View File

@@ -1,48 +0,0 @@
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.addColumn(
"RealEstateRequests",
"sizeRange",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"gardenSizeRange",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"priceRange",
{
type: Sequelize.STRING
},
{ transaction: t }
)
]);
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.removeColumn("RealEstateRequests", "sizeRange", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "gardenSizeRange", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "priceRange", {
transaction: t
})
]);
});
}
};

View File

@@ -1,147 +0,0 @@
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.removeColumn("RealEstateRequests", "sizeRange", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "gardenSizeRange", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "priceRange", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "size", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "gardenSize", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "price", {
transaction: t
}),
queryInterface.addColumn(
"RealEstateRequests",
"gardenSizeMin",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"gardenSizeMax",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"sizeMin",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"sizeMax",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"priceMin",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"priceMax",
{
type: Sequelize.INTEGER
},
{ transaction: t }
)
]);
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.removeColumn("RealEstateRequests", "gardenSizeMin", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "gardenSizeMax", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "sizeMin", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "sizeMax", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "priceMin", {
transaction: t
}),
queryInterface.removeColumn("RealEstateRequests", "priceMin", {
transaction: t
}),
queryInterface.addColumn(
"RealEstateRequests",
"priceMax",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"gardenSizeRange",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"priceRange",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"size",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"gardenSize",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"RealEstateRequests",
"price",
{
type: Sequelize.STRING
},
{ transaction: t }
)
]);
});
}
};

View File

@@ -1,15 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn(
"RealEstateRequests",
"subscribed",
Sequelize.BOOLEAN
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstateRequests", "subscribed");
}
};

View File

@@ -1,70 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.addColumn(
"MarketAlerts",
"size",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"MarketAlerts",
"gardenSize",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"MarketAlerts",
"price",
{
type: Sequelize.INTEGER
},
{ transaction: t }
),
queryInterface.addColumn(
"MarketAlerts",
"municipality",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"MarketAlerts",
"region",
{
type: Sequelize.STRING
},
{ transaction: t }
)
]);
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.removeColumn("MarketAlerts", "size", { transaction: t }),
queryInterface.removeColumn("MarketAlerts", "gardenSize", {
transaction: t
}),
queryInterface.removeColumn("MarketAlerts", "price", {
transaction: t
}),
queryInterface.removeColumn("MarketAlerts", "municipality", {
transaction: t
}),
queryInterface.removeColumn("MarketAlerts", "region", {
transaction: t
})
]);
});
}
};

View File

@@ -1,59 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.removeColumn("MarketAlerts", "olxUrl", {
transaction: t
}),
queryInterface.addColumn(
"MarketAlerts",
"url",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"MarketAlerts",
"realestateOrigin",
{
type: Sequelize.STRING
},
{ transaction: t }
),
queryInterface.addColumn(
"MarketAlerts",
"originId",
{
type: Sequelize.STRING
},
{ transaction: t }
)
]);
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(t => {
return Promise.all([
queryInterface.removeColumn("MarketAlerts", "url", { transaction: t }),
queryInterface.removeColumn("MarketAlerts", "realestateOrigin", {
transaction: t
}),
queryInterface.removeColumn("MarketAlerts", "originId", {
transaction: t
}),
queryInterface.addColumn(
"MarketAlerts",
"olxUrl",
{
type: Sequelize.STRING
},
{ transaction: t }
)
]);
});
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("MarketAlerts", "realEstateType", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("MarketAlerts", "realEstateType");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("MarketAlerts", "notified", {
type: Sequelize.BOOLEAN
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("MarketAlerts", "notified");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("MarketAlerts", "title", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("MarketAlerts", "title");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("MarketAlerts", "request", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("MarketAlerts", "request");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("MarketAlerts", "hasLocation", {
type: Sequelize.BOOLEAN
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("MarketAlerts", "hasLocation");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("RealEstateRequests", "locationInput", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstateRequests", "locationInput");
}
};

View File

@@ -1,19 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.renameColumn(
"RealEstateRequests",
"bounding_box",
"boundingBox"
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.renameColumn(
"RealEstateRequests",
"boundingBox",
"bounding_box"
);
}
};

View File

@@ -1,72 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
const tableFields = {
id: {
type: Sequelize.BIGINT,
autoIncrement: true,
allowNull: false,
primaryKey: true
},
url: {
type: Sequelize.TEXT,
allowNull: false
},
agencyObjectId: {
type: Sequelize.TEXT,
allowNull: false
},
originAgencyName: {
type: Sequelize.TEXT,
allowNull: false
},
realEstateType: {
type: Sequelize.TEXT,
allowNull: false
},
adType: {
type: Sequelize.TEXT,
allowNull: false
},
price: Sequelize.REAL,
area: Sequelize.REAL,
gardenSize: Sequelize.REAL,
streetNumber: Sequelize.INTEGER,
streetName: Sequelize.TEXT,
locality: Sequelize.TEXT,
municipality: Sequelize.TEXT,
city: Sequelize.TEXT,
region: Sequelize.TEXT,
entity: Sequelize.TEXT,
country: Sequelize.TEXT,
locationLat: Sequelize.REAL,
locationLong: Sequelize.REAL,
lastTimeCrawled: {
type: Sequelize.DATE,
allowNull: false
},
deleted: {
type: Sequelize.BOOLEAN,
allowNull: false
},
sold: {
type: Sequelize.BOOLEAN,
allowNull: false
},
createdAt: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal("NOW()")
},
updatedAt: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal("NOW()")
}
};
return queryInterface.createTable("RealEstates", tableFields);
},
down: queryInterface => {
return queryInterface.dropTable("RealEstates", {});
}
};

View File

@@ -1,79 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
const tableFields = {
id: {
type: Sequelize.UUID,
defaultValue: Sequelize.UUIDV4,
allowNull: false,
primaryKey: true
},
areaToSearch: {
type: Sequelize.GEOMETRY("POLYGON", 4326),
allowNull: false,
defaultValue: {
type: "Polygon",
coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
crs: { type: "name", properties: { name: "EPSG:4326" } }
}
},
realEstateType: {
type: Sequelize.TEXT,
allowNull: false
},
adType: {
type: Sequelize.TEXT,
allowNull: false,
defaultValue: "sell"
},
email: Sequelize.TEXT,
locality: Sequelize.TEXT,
municipality: Sequelize.TEXT,
city: Sequelize.TEXT,
region: Sequelize.TEXT,
entity: Sequelize.TEXT,
country: Sequelize.TEXT,
sizeMin: {
type: Sequelize.INTEGER,
allowNull: false,
defaultValue: 0
},
sizeMax: {
type: Sequelize.INTEGER,
allowNull: false,
defaultValue: 0
},
priceMin: {
type: Sequelize.INTEGER,
allowNull: false,
defaultValue: 0
},
priceMax: {
type: Sequelize.INTEGER,
allowNull: false,
defaultValue: 0
},
gardenSizeMin: Sequelize.INTEGER,
gardenSizeMax: Sequelize.INTEGER,
subscribed: {
type: Sequelize.BOOLEAN,
allowNull: false,
defaultValue: false
},
createdAt: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal("NOW()")
},
updatedAt: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal("NOW()")
}
};
return queryInterface.createTable("SearchRequests", tableFields);
},
down: queryInterface => {
return queryInterface.dropTable("SearchRequests", {});
}
};

View File

@@ -1,53 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
const tableFields = {
id: {
type: Sequelize.BIGINT,
autoIncrement: true,
allowNull: false
},
searchRequestId: {
type: Sequelize.UUID,
allowNull: false,
primaryKey: true,
references: {
model: "SearchRequests",
key: "id"
},
onUpdate: "CASCADE",
onDelete: "SET NULL"
},
realEstateId: {
type: Sequelize.BIGINT,
allowNull: false,
primaryKey: true,
references: {
model: "RealEstates",
key: "id"
},
onUpdate: "CASCADE",
onDelete: "SET NULL"
},
notified: {
type: Sequelize.BOOLEAN,
allowNull: false,
defaultValue: false
},
createdAt: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal("NOW()")
},
updatedAt: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal("NOW()")
}
};
return queryInterface.createTable("SearchRequestMatches", tableFields);
},
down: queryInterface => {
return queryInterface.dropTable("SearchRequestMatches", {});
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("RealEstates", "title", {
type: Sequelize.STRING
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstates", "title");
}
};

View File

@@ -1,21 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.addColumn("RealEstates", "shortDescription", {
type: Sequelize.STRING
}),
queryInterface.addColumn("RealEstates", "longDescription", {
type: Sequelize.STRING
})
]);
},
down: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.removeColumn("RealEstates", "shortDescription"),
queryInterface.removeColumn("RealEstates", "longDescription")
]);
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("RealEstates", "adStatus", {
type: Sequelize.INTEGER
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstates", "adStatus");
}
};

View File

@@ -1,21 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addConstraint(
"RealEstates",
["originAgencyName", "agencyObjectId"],
{
type: "unique",
name: "agencyNameObjectIdUniqueKey"
}
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeConstraint(
"RealEstates",
"agencyNameObjectIdUniqueKey"
);
}
};

View File

@@ -1,14 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("RealEstates", "lastTimeCrawled");
},
down: (queryInterface, Sequelize) => {
return queryInterface.addColumn("RealEstates", "lastTimeCrawled", {
type: Sequelize.DATE,
notNull: true
});
}
};

View File

@@ -1,23 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.removeColumn("RealEstates", "deleted"),
queryInterface.removeColumn("RealEstates", "sold")
]);
},
down: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.addColumn("RealEstates", "deleted", {
type: Sequelize.BOOLEAN,
notNull: true
}),
queryInterface.addColumn("RealEstates", "sold", {
type: Sequelize.BOOLEAN,
notNull: true
})
]);
}
};

View File

@@ -1,21 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.changeColumn("RealEstates", "shortDescription", {
type: Sequelize.TEXT
}),
queryInterface.changeColumn("RealEstates", "longDescription", {
type: Sequelize.TEXT
}),
queryInterface.changeColumn("RealEstates", "title", {
type: Sequelize.TEXT
})
]);
},
down: (queryInterface, Sequelize) => {
return Promise.all([]);
}
};

View File

@@ -1,21 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.addColumn("RealEstates", "publishedDate", {
type: Sequelize.DATE
}),
queryInterface.addColumn("RealEstates", "renewedDate", {
type: Sequelize.DATE
})
]);
},
down: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.removeColumn("RealEstates", "renewedDate"),
queryInterface.removeColumn("RealEstates", "publishedDate")
]);
}
};

View File

@@ -1,15 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "adType" = 'SALE' WHERE "adType" = 'sell';`
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "adType" = 'sell' WHERE "adType" = 'SALE';`
);
}
};

View File

@@ -1,31 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "realEstateType" = 'HOUSE' WHERE "realEstateType" = 'kuca';`
),
queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "realEstateType" = 'FLAT' WHERE "realEstateType" = 'stan';`
),
queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "realEstateType" = 'COTTAGE' WHERE "realEstateType" = 'vikendica';`
)
]);
},
down: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "realEstateType" = 'kuca' WHERE "realEstateType" = 'HOUSE';`
),
queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "realEstateType" = 'stan' WHERE "realEstateType" = 'FLAT';`
),
queryInterface.sequelize.query(
`UPDATE "SearchRequests" SET "realEstateType" = 'vikendica' WHERE "realEstateType" = 'COTTAGE';`
)
]);
}
};

View File

@@ -1,15 +0,0 @@
"use strict";
const { EMAIL_FREQUENCY } = require("../common/enums");
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("SearchRequests", "emailFrequency", {
type: Sequelize.TEXT,
defaultValue: EMAIL_FREQUENCY.ASAP.stringId
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("SearchRequests", "emailFrequency");
}
};

View File

@@ -1,13 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn("SearchRequests", "deletedEmail", {
type: Sequelize.TEXT
});
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn("SearchRequests", "deletedEmail");
}
};

View File

@@ -1,163 +0,0 @@
"use strict";
module.exports = {
up: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.addColumn("RealEstates", "numberOfRooms", {
type: Sequelize.REAL
}),
queryInterface.addColumn("RealEstates", "numberOfFloors", {
type: Sequelize.INTEGER
}),
queryInterface.addColumn("RealEstates", "floor", {
type: Sequelize.INTEGER
}),
queryInterface.addColumn("RealEstates", "accessRoadType", {
type: Sequelize.TEXT
}),
queryInterface.addColumn("RealEstates", "heatingType", {
type: Sequelize.TEXT
}),
queryInterface.addColumn("RealEstates", "furnishingType", {
type: Sequelize.TEXT
}),
queryInterface.addColumn("RealEstates", "balcony", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "newBuilding", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "elevator", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "water", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "electricity", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "drainageSystem", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "registeredInZkBooks", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "recentlyAdapted", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "parking", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "garage", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "gas", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "antiTheftDoor", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "airCondition", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "phoneConnection", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "cableTV", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "internet", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "basementAttic", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "storeRoom", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "videoSurveillance", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "alarm", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "suitableForStudents", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "includingBills", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "animalsAllowed", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "pool", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "exchange", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "urbanPlanPermit", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "buildingPermit", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "utilityConnection", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("RealEstates", "distanceToRiver", {
type: Sequelize.INTEGER
}),
queryInterface.addColumn("RealEstates", "numberOfViewsAgency", {
type: Sequelize.INTEGER,
defaultValue: 0
}),
queryInterface.addColumn("RealEstates", "numberOfViewsKivi", {
type: Sequelize.INTEGER,
defaultValue: 0
})
]);
},
down: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.removeColumn("RealEstates", "numberOfRooms"),
queryInterface.removeColumn("RealEstates", "numberOfFloors"),
queryInterface.removeColumn("RealEstates", "floor"),
queryInterface.removeColumn("RealEstates", "accessRoadType"),
queryInterface.removeColumn("RealEstates", "heatingType"),
queryInterface.removeColumn("RealEstates", "furnishingType"),
queryInterface.removeColumn("RealEstates", "balcony"),
queryInterface.removeColumn("RealEstates", "newBuilding"),
queryInterface.removeColumn("RealEstates", "elevator"),
queryInterface.removeColumn("RealEstates", "water"),
queryInterface.removeColumn("RealEstates", "electricity"),
queryInterface.removeColumn("RealEstates", "drainageSystem"),
queryInterface.removeColumn("RealEstates", "registeredInZkBooks"),
queryInterface.removeColumn("RealEstates", "recentlyAdapted"),
queryInterface.removeColumn("RealEstates", "parking"),
queryInterface.removeColumn("RealEstates", "garage"),
queryInterface.removeColumn("RealEstates", "gas"),
queryInterface.removeColumn("RealEstates", "antiTheftDoor"),
queryInterface.removeColumn("RealEstates", "airCondition"),
queryInterface.removeColumn("RealEstates", "phoneConnection"),
queryInterface.removeColumn("RealEstates", "cableTV"),
queryInterface.removeColumn("RealEstates", "internet"),
queryInterface.removeColumn("RealEstates", "basementAttic"),
queryInterface.removeColumn("RealEstates", "storeRoom"),
queryInterface.removeColumn("RealEstates", "videoSurveillance"),
queryInterface.removeColumn("RealEstates", "alarm"),
queryInterface.removeColumn("RealEstates", "suitableForStudents"),
queryInterface.removeColumn("RealEstates", "includingBills"),
queryInterface.removeColumn("RealEstates", "animalsAllowed"),
queryInterface.removeColumn("RealEstates", "pool"),
queryInterface.removeColumn("RealEstates", "exchange"),
queryInterface.removeColumn("RealEstates", "urbanPlanPermit"),
queryInterface.removeColumn("RealEstates", "buildingPermit"),
queryInterface.removeColumn("RealEstates", "utilityConnection"),
queryInterface.removeColumn("RealEstates", "distanceToRiver"),
queryInterface.removeColumn("RealEstates", "numberOfViewsAgency"),
queryInterface.removeColumn("RealEstates", "numberOfViewsKivi")
]);
}
};

View File

@@ -1,64 +0,0 @@
"use strict";
const { ACCESS_ROAD_TYPE, HEATING_TYPE } = require("../common/enums");
module.exports = {
up: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.addColumn("SearchRequests", "includeIncompleteAds", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("SearchRequests", "balcony", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("SearchRequests", "newBuilding", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("SearchRequests", "elevator", {
type: Sequelize.BOOLEAN
}),
queryInterface.addColumn("SearchRequests", "numberOfRoomsMin", {
type: Sequelize.REAL
}),
queryInterface.addColumn("SearchRequests", "numberOfRoomsMax", {
type: Sequelize.REAL
}),
queryInterface.addColumn("SearchRequests", "numberOfFloorsMin", {
type: Sequelize.INTEGER
}),
queryInterface.addColumn("SearchRequests", "numberOfFloorsMax", {
type: Sequelize.INTEGER
}),
queryInterface.addColumn("SearchRequests", "floorMin", {
type: Sequelize.INTEGER
}),
queryInterface.addColumn("SearchRequests", "floorMax", {
type: Sequelize.INTEGER
}),
queryInterface.addColumn("SearchRequests", "accessRoadType", {
type: Sequelize.TEXT,
defaultValue: ACCESS_ROAD_TYPE.ANY.id
}),
queryInterface.addColumn("SearchRequests", "heatingType", {
type: Sequelize.TEXT,
defaultValue: HEATING_TYPE.ANY.id
})
]);
},
down: (queryInterface, Sequelize) => {
return Promise.all([
queryInterface.removeColumn("SearchRequests", "includeIncompleteAds"),
queryInterface.removeColumn("SearchRequests", "balcony"),
queryInterface.removeColumn("SearchRequests", "newBuilding"),
queryInterface.removeColumn("SearchRequests", "elevator"),
queryInterface.removeColumn("SearchRequests", "numberOfRoomsMin"),
queryInterface.removeColumn("SearchRequests", "numberOfRoomsMax"),
queryInterface.removeColumn("SearchRequests", "numberOfFloorsMin"),
queryInterface.removeColumn("SearchRequests", "numberOfFloorsMax"),
queryInterface.removeColumn("SearchRequests", "floorMin"),
queryInterface.removeColumn("SearchRequests", "floorMax"),
queryInterface.removeColumn("SearchRequests", "accessRoadType"),
queryInterface.removeColumn("SearchRequests", "heatingType")
]);
}
};

View File

@@ -1,49 +0,0 @@
"use strict";
const fs = require("fs");
const path = require("path");
const Sequelize = require("sequelize");
const basename = path.basename(__filename);
const env = process.env.NODE_ENV || "development";
const config = require(__dirname + "/../config/config.json")[env];
const db = {};
config.username = process.env.DB_USERNAME || config.username;
config.password = process.env.DB_PASSWORD || config.password;
config.database = process.env.DB_NAME || config.database;
config.port = process.env.DB_PORT || config.port;
config.logging = parseInt(process.env.SEQUELIZE_LOGGING) ? console.log : false;
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(
config.database,
config.username,
config.password,
config
);
}
fs.readdirSync(__dirname)
.filter(file => {
return (
file.indexOf(".") !== 0 && file !== basename && file.slice(-3) === ".js"
);
})
.forEach(file => {
const model = sequelize["import"](path.join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;

View File

@@ -1,92 +0,0 @@
"use strict";
module.exports = (sequelize, DataTypes) => {
const RealEstate = sequelize.define("RealEstate", {
id: {
type: DataTypes.BIGINT,
autoIncrement: true,
allowNull: false,
primaryKey: true
},
url: {
type: DataTypes.TEXT,
allowNull: false
},
originAgencyName: {
type: DataTypes.TEXT,
allowNull: false,
unique: true
},
agencyObjectId: {
type: DataTypes.TEXT,
allowNull: false,
unique: true
},
realEstateType: {
type: DataTypes.TEXT,
allowNull: false
},
adType: {
type: DataTypes.TEXT,
allowNull: false
},
price: DataTypes.REAL,
area: DataTypes.REAL,
gardenSize: DataTypes.REAL,
streetNumber: DataTypes.INTEGER,
streetName: DataTypes.TEXT,
locality: DataTypes.TEXT,
municipality: DataTypes.TEXT,
city: DataTypes.TEXT,
region: DataTypes.TEXT,
entity: DataTypes.TEXT,
country: DataTypes.TEXT,
locationLat: DataTypes.REAL,
locationLong: DataTypes.REAL,
title: DataTypes.TEXT,
shortDescription: DataTypes.TEXT,
longDescription: DataTypes.TEXT,
adStatus: DataTypes.INTEGER,
publishedDate: DataTypes.DATE,
renewedDate: DataTypes.DATE,
numberOfRooms: DataTypes.INTEGER,
numberOfFloors: DataTypes.INTEGER,
floor: DataTypes.INTEGER,
accessRoadType: DataTypes.TEXT,
heatingType: DataTypes.TEXT,
furnishingType: DataTypes.TEXT,
balcony: DataTypes.BOOLEAN,
newBuilding: DataTypes.BOOLEAN,
elevator: DataTypes.BOOLEAN,
water: DataTypes.BOOLEAN,
electricity: DataTypes.BOOLEAN,
drainageSystem: DataTypes.BOOLEAN,
registeredInZkBooks: DataTypes.BOOLEAN,
recentlyAdapted: DataTypes.BOOLEAN,
parking: DataTypes.BOOLEAN,
garage: DataTypes.BOOLEAN,
gas: DataTypes.BOOLEAN,
antiTheftDoor: DataTypes.BOOLEAN,
airCondition: DataTypes.BOOLEAN,
phoneConnection: DataTypes.BOOLEAN,
cableTV: DataTypes.BOOLEAN,
internet: DataTypes.BOOLEAN,
basementAttic: DataTypes.BOOLEAN,
storeRoom: DataTypes.BOOLEAN,
videoSurveillance: DataTypes.BOOLEAN,
alarm: DataTypes.BOOLEAN,
suitableForStudents: DataTypes.BOOLEAN,
includingBills: DataTypes.BOOLEAN,
animalsAllowed: DataTypes.BOOLEAN,
pool: DataTypes.BOOLEAN,
exchange: DataTypes.BOOLEAN,
urbanPlanPermit: DataTypes.BOOLEAN,
buildingPermit: DataTypes.BOOLEAN,
utilityConnection: DataTypes.BOOLEAN,
distanceToRiver: DataTypes.INTEGER,
numberOfViewsAgency: DataTypes.INTEGER,
numberOfViewsKivi: DataTypes.INTEGER
});
return RealEstate;
};

View File

@@ -1,88 +0,0 @@
"use strict";
const { AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
module.exports = (sequelize, DataTypes) => {
const SearchRequest = sequelize.define("SearchRequest", {
id: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
allowNull: false,
primaryKey: true
},
areaToSearch: {
type: DataTypes.GEOMETRY("POLYGON", 4326),
allowNull: false,
defaultValue: {
type: "Polygon",
coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
crs: { type: "name", properties: { name: "EPSG:4326" } }
}
},
realEstateType: {
type: DataTypes.TEXT,
allowNull: false
},
adType: {
type: DataTypes.TEXT,
allowNull: false,
defaultValue: AD_TYPE.AD_TYPE_SALE.stringId
},
email: DataTypes.TEXT,
locality: DataTypes.TEXT,
municipality: DataTypes.TEXT,
city: DataTypes.TEXT,
region: DataTypes.TEXT,
entity: DataTypes.TEXT,
country: DataTypes.TEXT,
sizeMin: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0
},
sizeMax: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0
},
priceMin: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0
},
priceMax: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0
},
gardenSizeMin: DataTypes.INTEGER,
gardenSizeMax: DataTypes.INTEGER,
subscribed: {
type: DataTypes.BOOLEAN,
defaultValue: false,
allowNull: false
},
emailFrequency: {
type: DataTypes.TEXT,
defaultValue: EMAIL_FREQUENCY.ASAP.stringId,
allowNull: false
},
deletedEmail: {
type: DataTypes.TEXT
},
includeIncompleteAds: DataTypes.BOOLEAN,
balcony: DataTypes.BOOLEAN,
elevator: DataTypes.BOOLEAN,
newBuilding: DataTypes.BOOLEAN,
numberOfRoomsMin: DataTypes.REAL,
numberOfRoomsMax: DataTypes.REAL,
numberOfFloorsMin: DataTypes.INTEGER,
numberOfFloorsMax: DataTypes.INTEGER,
floorMin: DataTypes.INTEGER,
floorMax: DataTypes.INTEGER,
accessRoadType: DataTypes.TEXT,
heatingType: DataTypes.TEXT
});
return SearchRequest;
};

View File

@@ -1,60 +0,0 @@
"use strict";
module.exports = (sequelize, DataTypes) => {
const SearchRequestMatch = sequelize.define(
"SearchRequestMatch",
{
id: {
type: DataTypes.BIGINT,
autoIncrement: true,
allowNull: false
},
realEstateId: {
type: DataTypes.BIGINT,
allowNull: false,
primaryKey: true,
references: {
model: "RealEstate",
key: "id"
},
onUpdate: "CASCADE",
onDelete: "SET NULL"
},
searchRequestId: {
type: DataTypes.UUID,
allowNull: false,
primaryKey: true,
references: {
model: "SearchRequest",
key: "id"
}
},
notified: {
type: DataTypes.BOOLEAN,
allowNull: false,
defaultValue: false
}
},
{
name: {
singular: "searchRequestMatch",
plural: "searchRequestMatches"
}
}
);
SearchRequestMatch.associate = models => {
SearchRequestMatch.hasMany(models.SearchRequest, {
foreignKey: "id",
sourceKey: "searchRequestId",
targetKey: "id",
as: "searchRequests"
});
SearchRequestMatch.hasMany(models.RealEstate, {
foreignKey: "id",
as: "realEstates"
});
};
return SearchRequestMatch;
};

View File

@@ -1,8 +0,0 @@
"use strict";
const {
notifyRequestsWithDailyOption
} = require("../services/notificationService");
(async () => {
await notifyRequestsWithDailyOption();
})();

View File

@@ -1,32 +0,0 @@
"use strict";
module.exports = (sequelize, DataTypes) => {
const MarketAlert = sequelize.define(
"MarketAlert",
{
url: DataTypes.STRING,
realestateOrigin: DataTypes.STRING,
originId: DataTypes.STRING,
lastDate: DataTypes.STRING,
size: DataTypes.INTEGER,
gardenSize: DataTypes.INTEGER,
price: DataTypes.INTEGER,
municipality: DataTypes.STRING,
region: DataTypes.STRING,
realEstateType: DataTypes.STRING,
notified: DataTypes.BOOLEAN,
title: DataTypes.STRING,
request: DataTypes.STRING,
hasLocation: DataTypes.BOOLEAN,
email: {
type: DataTypes.STRING,
allowNul: false
}
},
{}
);
MarketAlert.associate = function(models) {
// associations can be defined here
};
return MarketAlert;
};

View File

@@ -1,32 +0,0 @@
"use strict";
module.exports = (sequelize, DataTypes) => {
const RealEstateRequest = sequelize.define(
"RealEstateRequest",
{
uniqueId: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
allowNull: false
},
realEstateType: DataTypes.STRING,
email: DataTypes.STRING,
region: DataTypes.STRING,
municipality: DataTypes.STRING,
sizeMin: DataTypes.INTEGER,
sizeMax: DataTypes.INTEGER,
gardenSizeMin: DataTypes.INTEGER,
gardenSizeMax: DataTypes.INTEGER,
priceMin: DataTypes.INTEGER,
priceMax: DataTypes.INTEGER,
boundingBox: DataTypes.GEOMETRY("POINT", 4326),
subscribed: DataTypes.BOOLEAN,
locationInput: DataTypes.STRING
},
{}
);
RealEstateRequest.associate = function(models) {
// associations can be defined here
};
return RealEstateRequest;
};

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.0 KiB

View File

@@ -1,9 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<browserconfig>
<msapplication>
<tile>
<square150x150logo src="/mstile-150x150.png"/>
<TileColor>#da532c</TileColor>
</tile>
</msapplication>
</browserconfig>

Binary file not shown.

Before

Width:  |  Height:  |  Size: 897 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.2 MiB

View File

@@ -1,122 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="61"
height="94"
viewBox="0 0 61 94"
version="1.1"
id="svg29"
sodipodi:docname="logo2.svg"
inkscape:version="0.92.3 (2405546, 2018-03-11)">
<metadata
id="metadata33">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1853"
inkscape:window-height="1025"
id="namedview31"
showgrid="false"
inkscape:zoom="5.6568543"
inkscape:cx="29.58234"
inkscape:cy="42.092869"
inkscape:window-x="67"
inkscape:window-y="27"
inkscape:window-maximized="1"
inkscape:current-layer="svg29" />
<!-- Generator: Sketch 51.3 (57544) - http://www.bohemiancoding.com/sketch -->
<desc
id="desc2">Created with Sketch.</desc>
<defs
id="defs4" />
<g
id="g938"
transform="translate(4.6566166)">
<g
id="Group"
transform="translate(21.468225,75.05246)"
style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1">
<path
style="fill:#02adba;fill-opacity:1"
d="m 2.8937203,1.377828 h 0.00886 V 16.18948 h -0.00886 c -0.080215,0.775019 -0.6954121,1.377828 -1.4424314,1.377828 -0.74701929,0 -1.36221635,-0.602809 -1.4424314,-1.377828 H 0 V 1.377828 H 0.0088575 C 0.08907255,0.60280831 0.70426961,0 1.4512889,0 2.1983082,0 2.8135053,0.60280831 2.8937203,1.377828 Z"
id="Combined-Shape"
inkscape:connector-curvature="0" />
</g>
<g
id="Group-Copy"
transform="translate(40.284746,75.05246)"
style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1">
<path
style="fill:#02adba;fill-opacity:1"
d="m 2.8937203,1.377828 h 0.00886 V 16.18948 h -0.00886 c -0.080215,0.775019 -0.6954121,1.377828 -1.4424314,1.377828 -0.74701929,0 -1.36221635,-0.602809 -1.4424314,-1.377828 H 0 V 1.377828 H 0.0088575 C 0.08907255,0.60280831 0.70426961,0 1.4512889,0 2.1983082,0 2.8135053,0.60280831 2.8937203,1.377828 Z"
id="path8"
inkscape:connector-curvature="0" />
</g>
<g
id="Group-2"
transform="translate(26.045022,75.05246)"
style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1">
<path
style="fill:#02adba;fill-opacity:1"
sodipodi:nodetypes="cccsscccsscccsc"
d="M 4.7335014,16.043249 0.01556995,2.0709177 v 0 C -0.03604093,1.9151618 -0.08984375,1.7196494 -0.08984375,1.5461504 -0.08984375,0.69007771 0.68116079,0 1.5214152,0 2.3524826,0 2.7388538,0.67107889 3.0425868,1.5220354 L 6.255723,11.044343 9.4667171,1.5292951 C 9.7627803,0.71794799 10.154547,0 10.987999,0 c 0.840254,0 1.603446,0.68226521 1.603446,1.5383379 0,0.1793218 -0.04867,0.3879772 -0.103645,0.5481862 v 0 L 7.776097,16.026251 c -0.6976248,1.816338 -0.6840821,1.541057 -1.5213901,1.541057 -0.831732,0 -0.9926068,0.05035 -1.5212055,-1.524059 z"
id="path11"
inkscape:connector-curvature="0" />
</g>
<path
sodipodi:nodetypes="ccccsscccsscccccscscccssscc"
inkscape:connector-curvature="0"
id="path14"
d="m 12.013315,76.606127 v 4.050014 l 4.717498,-5.019565 v 0 c 0.404985,-0.305017 0.852764,-0.412505 1.290558,-0.412505 0.899311,0 1.590893,0.796979 1.590893,1.653051 0,0.589329 -0.373586,0.895853 -0.854196,1.364015 l -5.796302,6.120383 5.715392,5.649676 c 0.532765,0.486219 0.935106,0.782903 0.935106,1.402972 0,0.856073 -0.729035,1.550057 -1.628346,1.550057 -0.420934,0 -0.861077,0.0072 -1.216312,-0.275429 v 0 l -4.754291,-4.690453 v 3.249235 h -0.0031 c 0.0021,0.03199 0.0031,0.06425 0.0031,0.09674 0,0.856072 -0.729035,1.550056 -1.628346,1.550056 -0.8993109,-10e-7 -1.6283459,-0.693985 -1.6283459,-1.550057 0,-0.03249 10e-4,-0.06475 0.0031,-0.09674 h -0.0031 v -14.64145 -0.0036 c 0,-0.856083 0.729035,-1.550067 1.6283459,-1.550067 0.899311,0 1.628346,0.693984 1.628346,1.550057 0,0.0012 -1e-6,0.0024 -4e-6,0.0036 z"
style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1" />
</g>
<g
style="fill:#02adba;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1"
id="Group-5-Copy"
transform="translate(0.30033447)">
<path
inkscape:connector-curvature="0"
id="Page-1-Copy-3"
d="m 28.864505,0.71910556 c 0.913861,-0.49697651 2.013643,-0.49697651 2.927503,0 L 58.602992,15.299478 c 1.00074,0.544224 1.624856,1.599801 1.624856,2.748147 V 52.95707 c 0,1.148497 -0.624278,2.204188 -1.625223,2.748347 L 31.791641,70.28107 c -0.913667,0.49671 -2.013102,0.49671 -2.926768,0 L 2.053889,55.705417 C 1.052944,55.161258 0.42866553,54.105567 0.42866553,52.95707 V 18.047625 c 0,-1.148346 0.62411557,-2.203923 1.62485607,-2.748147 z M 30.328257,3.4672523 3.5172731,18.047625 V 52.95707 L 30.328257,67.532724 57.13924,52.95707 V 18.047625 Z"
style="fill:#02adba;fill-opacity:1" />
<path
inkscape:connector-curvature="0"
transform="matrix(-1,0,0,1,38.607594,0)"
id="Page-1-Copy-2"
d="m 6.6881981,8.5296646 c 0,-1.1837877 1.2534136,-1.9364737 2.2824613,-1.3706383 L 31.113251,19.334421 c 0.496869,0.273211 0.806146,0.799054 0.806146,1.370639 v 29.035897 c 0,0.571661 -0.309358,1.097561 -0.806331,1.37074 L 8.970475,63.283151 C 7.9414326,63.848801 6.6881981,63.096106 6.6881981,61.912412 Z M 9.7768057,11.155344 V 59.287138 L 28.830789,48.813446 V 21.632428 Z"
style="fill:#02adba;fill-opacity:1" />
<path
inkscape:connector-curvature="0"
transform="rotate(4,36.513012,12.39682)"
id="Path-85"
d="m 29.442604,10.590184 12.454251,6.520214 c 0.879716,0.424975 1.970417,0.118743 2.436149,-0.683987 0.465733,-0.802729 0.130132,-1.797981 -0.749584,-2.222955 L 31.12917,7.6832415 C 30.249453,7.2582668 29.158752,7.564498 28.69302,8.367228 c -0.465732,0.8027299 -0.130132,1.797981 0.749584,2.222956 z"
style="fill:#02adba;fill-opacity:1" />
</g>
<path
style="fill:#02adba;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1"
inkscape:connector-curvature="0"
id="Path-87"
d="m 7.8821311,47.17753 c -0.7181691,0.486659 -0.8975584,1.451387 -0.4006772,2.15478 0.4968812,0.703394 1.4818741,0.879093 2.2000432,0.392434 L 31.518752,34.926994 c 0.939679,-0.636761 0.900745,-2.009481 -0.07358,-2.594168 L 9.6079198,19.228381 c -0.744646,-0.446859 -1.718161,-0.217874 -2.1744066,0.511452 -0.4562456,0.729326 -0.2224509,1.682812 0.5221952,2.129671 L 27.723653,33.732165 Z" />
</svg>

Before

Width:  |  Height:  |  Size: 7.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 216 KiB

View File

@@ -1,156 +0,0 @@
@font-face {
font-family: "Alte Haas Grotesk";
src: url("./fonts/altehaasgroteskregular.ttf");
}
@font-face {
font-family: "Alte Haas Grotesk";
src: url("./fonts/altehaasgroteskbold.ttf");
font-weight: bold;
}
.welcome-center-button {
width: 100%;
}
.next-center-button {
width: 50%;
left: 25%;
}
.no-ui-slider {
width: 95%;
}
.noUi-connect {
background: #02adba;
}
.centered-element {
margin-top: 200px;
}
.centered-element-small {
margin-top: 100px;
}
.btn,
.btn-floating {
background: #02adba;
font-weight: bold;
}
.kivi-color {
color: #02adba;
}
.kivi-spinner-color {
border-color: #02adba;
}
#map {
height: 50%;
}
html,
body {
height: 100%;
margin: 0;
padding: 0;
font-family: "Alte Haas Grotesk", serif;
box-sizing: border-box;
}
.btn:hover {
background-color: white;
color: #02adba;
border: 1px solid rgb(0, 173, 187);
}
.btn-floating:hover {
background-color: #02adba;
border: 1px solid rgb(0, 173, 187);
}
h6.title {
margin-top: 0;
font-weight: bold;
padding-top: 20px;
font-size: 1.3rem;
}
.locate-me-container {
margin-right: 10px;
}
.full-width {
width: 100%;
}
strong {
font-weight: bold;
}
h3 {
font-size: 15px;
line-height: 1.5;
}
.sliderInputBox {
box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.4) !important;
border: 1px solid #02adba !important;
border-radius: 4px !important;
text-align: center;
}
.collection a.collection-item {
color: #02adba;
}
.collection a.collection-item:not(.active):hover {
background-color: rgba(2, 173, 186, 0.2);
}
.tabs .tab a {
color: #02adba;
-webkit-transition: color 0.28s ease, background-color 0.28s ease;
transition: color 0.28s ease, background-color 0.28s ease;
}
.tabs .tab a:focus,
.tabs .tab a:focus.active {
background-color: rgba(2, 173, 186, 0.2);
}
.tabs .tab a:hover,
.tabs .tab a.active {
color: #02adba;
}
.tabs .indicator {
background-color: #02adba;
}
[type="checkbox"].filled-in:checked + span:not(.lever):after {
border: 2px solid #02adba;
background-color: #02adba;
}
[type="checkbox"].filled-in:not(:checked) + span:not(.lever):after {
background-color: transparent;
border: 2px solid #02adba;
}
.distinguished {
border: 2px solid #02adba;
border-radius: 4px;
padding: 5px 5px 3px 5px;
margin-left: -5px;
}
.checkbox-label {
color: black;
font-size: 14px;
}
.column-label {
position: relative;
margin-top: 2rem;
margin-bottom: 1rem;
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.9 KiB

Some files were not shown because too many files have changed in this diff Show More