diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..82435b1 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +*.ejs \ No newline at end of file diff --git a/app/common/enums.js b/app/common/enums.js index 85ed553..20aa4dd 100644 --- a/app/common/enums.js +++ b/app/common/enums.js @@ -7,7 +7,42 @@ const PRICE_SLIDER_OPTIONS_SALE = { step: 1000, connect: true }; - +const FLAT_PRICE_SLIDER_OPTIONS_SALE = { + start: [50000, 150000], + range: { + min: [0], + max: [800000] + }, + step: 5000, + connect: true +}; +const HOUSE_PRICE_SLIDER_OPTIONS_SALE = { + start: [50000, 150000], + range: { + min: [0], + max: [1500000] + }, + step: 10000, + connect: true +}; +const OFFICE_PRICE_SLIDER_OPTIONS_SALE = { + start: [15000, 50000], + range: { + min: [0], + max: [2000000] + }, + step: 2000, + connect: true +}; +const LAND_PRICE_SLIDER_OPTIONS_SALE = { + start: [40000, 80000], + range: { + min: [0], + max: [2000000] + }, + step: 10000, + connect: true +}; const PRICE_SLIDER_OPTIONS_RENT = { start: [300, 500], range: { @@ -17,18 +52,62 @@ const PRICE_SLIDER_OPTIONS_RENT = { step: 50, connect: true }; - +const FLAT_PRICE_SLIDER_OPTIONS_RENT = { + start: [300, 600], + range: { + min: [0], + max: [4000] + }, + step: 100, + connect: true +}; +const HOUSE_PRICE_SLIDER_OPTIONS_RENT = { + start: [500, 1000], + range: { + min: [0], + max: [10000] + }, + step: 100, + connect: true +}; +const OFFICE_PRICE_SLIDER_OPTIONS_RENT = { + start: [200, 1000], + range: { + min: [0], + max: [20000] + }, + step: 100, + connect: true +}; +const LAND_PRICE_SLIDER_OPTIONS_RENT = { + start: [500, 1000], + range: { + min: [0], + max: [20000] + }, + step: 100, + connect: true +}; //This will be used for Flats, Apartments, Houses const HOME_SIZE_SLIDER_OPTIONS = { start: [30, 75], range: { min: [0], - max: [400] + max: [500] }, step: 5, connect: true }; +const OFFICE_SIZE_SLIDER_OPTIONS = { + start: [30, 150], + range: { + min: [0], + max: [1200] + }, + step: 10, + connect: true +}; const GARDEN_SIZE_SLIDER_OPTIONS = { start: [100, 1000], range: { @@ -111,8 +190,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: false, hasFloorProp: true, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS }, HOUSE: { @@ -126,8 +205,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: true, hasFloorProp: false, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS, gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS }, @@ -142,9 +221,9 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: false, hasFloorProp: true, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, - sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS + priceSliderOptionsSale: OFFICE_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: OFFICE_PRICE_SLIDER_OPTIONS_RENT, + sizeSliderOptions: OFFICE_SIZE_SLIDER_OPTIONS }, LAND: { id: "LAND", @@ -157,8 +236,8 @@ const AD_CATEGORY = { hasNumberOfRoom: false, hasNumberOfFloors: false, hasFloorProp: false, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: LAND_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: LAND_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS }, APARTMENT: { @@ -172,8 +251,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: false, hasFloorProp: true, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS }, GARAGE: { @@ -202,8 +281,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: true, hasFloorProp: false, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS, gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS } diff --git a/app/config/appConfig.js b/app/config/appConfig.js index b4310e0..3e6003f 100644 --- a/app/config/appConfig.js +++ b/app/config/appConfig.js @@ -16,6 +16,8 @@ const DEFAULT_TIMEZONE = "Europe/Sarajevo"; const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60; const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER); +const CHECK_UP_DAYS = parseInt(process.env.CHECK_UP_DAYS) || 10; + const AWS_EMAIL_CONFIG = { REGION: process.env.AWS_REGION || "", CREDENTIALS: { @@ -50,6 +52,8 @@ module.exports = { MAX_REAL_ESTATES_IN_FIRST_EMAIL, PRINT_CRAWLER_DEBUG, API_MAP_KEY, - PROSTOR_LOGIN, - STAGING + STAGING, + CHECK_UP_DAYS, + PROSTOR_LOGIN + }; diff --git a/app/controllers/realEstateFilters.js b/app/controllers/realEstateFilters.js index 17f5e58..252ac46 100644 --- a/app/controllers/realEstateFilters.js +++ b/app/controllers/realEstateFilters.js @@ -35,7 +35,8 @@ const getFilters = async (req, res) => { balcony, elevator, newBuilding, - accessRoadType + accessRoadType, + includeWithoutPrice } = searchRequest; const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT; @@ -115,7 +116,8 @@ const getFilters = async (req, res) => { advancedSegmentSelectFilterValues, advancedRangeFilterObjects, advancedRangeFilterValues, - includeIncompleteAds + includeIncompleteAds, + includeWithoutPrice }); }; @@ -191,6 +193,7 @@ const postFilters = async (req, res) => { }); const includeIncompleteAds = req.body.includeIncompleteAds === "on"; + const includeWithoutPrice = req.body.includeWithoutPrice === "on"; const balcony = req.body.balcony === "on"; const elevator = req.body.elevator === "on"; @@ -217,6 +220,7 @@ const postFilters = async (req, res) => { searchRequest.newBuilding = newBuilding; searchRequest.includeIncompleteAds = includeIncompleteAds; + searchRequest.includeWithoutPrice = includeWithoutPrice; searchRequest.accessRoadType = accessRoadType; diff --git a/app/crawler/savers/postgres.js b/app/crawler/savers/postgres.js index 344e4ac..97ba285 100644 --- a/app/crawler/savers/postgres.js +++ b/app/crawler/savers/postgres.js @@ -1,6 +1,7 @@ const moment = require("moment"); const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate"); +const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory"); class PostgresSaver { connect() { @@ -11,6 +12,21 @@ class PostgresSaver { async save(results) { const savedRecords = await bulkUpsertRealEstates(results); + //Extruding data for price history table + const resultPrices = savedRecords.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + + return { + realEstateId: realEstate.dataValues.id, + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt + }; + }); + const savedPrices = await bulkUpsertPriceHistory(resultPrices); if (Array.isArray(savedRecords)) { const newRealEstates = []; diff --git a/app/helpers/db/priceHistory.js b/app/helpers/db/priceHistory.js new file mode 100644 index 0000000..3e8f69e --- /dev/null +++ b/app/helpers/db/priceHistory.js @@ -0,0 +1,20 @@ +"use strict"; +const db = require("../../models/index"); +const sequelize = require("sequelize"); + +const bulkUpsertPriceHistory = async priceHistoryData => { + try { + const order = [["realEstateId", "desc"]]; + + return await db.PriceHistory.bulkCreate(priceHistoryData, { + order, + ignoreDuplicates: true + }); + } catch (e) { + console.log("Error bulk upserting priceHistory : ", e); + } +}; + +module.exports = { + bulkUpsertPriceHistory +}; diff --git a/app/helpers/db/realEstate.js b/app/helpers/db/realEstate.js index ebeb84c..0f77260 100644 --- a/app/helpers/db/realEstate.js +++ b/app/helpers/db/realEstate.js @@ -2,6 +2,8 @@ const db = require("../../models/index"); const sequelize = require("sequelize"); const Op = sequelize.Op; +const { AD_CATEGORY } = require("../../common/enums"); + const bulkUpsertRealEstates = async realEstateData => { try { const fieldsToUpdateIfDuplicate = [ @@ -96,12 +98,16 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { floorMin, floorMax, includeIncompleteAds, + includeWithoutPrice, balcony, elevator, newBuilding, accessRoadType } = searchRequest; + //Needed for defining which attribute should exist or not + const realEstateTypeObject = AD_CATEGORY[realEstateType]; + const longitudeColumn = sequelize.col("locationLong"); const latitudeColumn = sequelize.col("locationLat"); @@ -134,15 +140,6 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { const query = { adType, realEstateType, - price: { - [Op.or]: { - [Op.and]: { - [Op.lte]: priceMax, - [Op.gte]: priceMin - }, - [Op.is]: null - } - }, area: { [Op.lte]: sizeMax, [Op.gte]: sizeMin @@ -154,15 +151,6 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { const queryIncludeIncomplete = { adType, realEstateType, - price: { - [Op.or]: { - [Op.and]: { - [Op.lte]: priceMax, - [Op.gte]: priceMin - }, - [Op.is]: null - } - }, area: { [Op.or]: { [Op.and]: { @@ -175,8 +163,49 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { [Op.and]: geoSearchQueryPart }; - //Every other attribute is checked separately and included in query only if it is defined - if (gardenSizeMax && gardenSizeMin) { + //Is user unchecked includeWithoutPrice FALSE then it shouldn't return null values of price + //If not then null values are accepted (this is DEFAULT) + //includeIncpompleteAds does not have effect on price query + if (includeWithoutPrice) { + query.price = { + [Op.or]: { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + }, + [Op.is]: null + } + }; + queryIncludeIncomplete.price = { + [Op.or]: { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + }, + [Op.is]: null + } + }; + } else { + query.price = { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + } + }; + queryIncludeIncomplete.price = { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + } + }; + } + //Every other attribute is checked separately and included in query only if it is defined for real estate type + + if ( + realEstateTypeObject.hasGardenSize && + gardenSizeMax != null && + gardenSizeMin != null + ) { query.gardenSize = { [Op.lte]: gardenSizeMax, [Op.gte]: gardenSizeMin @@ -192,7 +221,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (numberOfRoomsMin && numberOfRoomsMax) { + if ( + realEstateTypeObject.hasNumberOfRoom && + numberOfRoomsMin != null && + numberOfRoomsMax != null + ) { query.numberOfRooms = { [Op.lte]: numberOfRoomsMax, [Op.gte]: numberOfRoomsMin @@ -208,7 +241,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (numberOfFloorsMin && numberOfFloorsMax) { + if ( + realEstateTypeObject.hasNumberOfFloors && + numberOfFloorsMin != null && + numberOfFloorsMax != null + ) { query.numberOfFloors = { [Op.lte]: numberOfFloorsMax, [Op.gte]: numberOfFloorsMin @@ -224,7 +261,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (floorMin && floorMax) { + if ( + realEstateTypeObject.hasFloorProp && + floorMin != null && + floorMax != null + ) { query.floor = { [Op.lte]: floorMax, [Op.gte]: floorMin @@ -239,8 +280,10 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { } }; } - - if (balcony) { + //Logic for balcony, newBuilding and elevator from users side + //If true is checked, then I want characteristic to be true but, + //if it is not checked, then I dont care - it can be null or false or true + if (realEstateTypeObject.hasBalconyProp && balcony === true) { query.balcony = { [Op.eq]: balcony }; @@ -252,7 +295,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (newBuilding) { + if (realEstateTypeObject.hasNewBuildingProp && newBuilding === true) { query.newBuilding = { [Op.eq]: newBuilding }; @@ -264,7 +307,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (elevator) { + if (realEstateTypeObject.hasElevatorProp && elevator === true) { query.elevator = { [Op.eq]: elevator }; @@ -275,7 +318,8 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { } }; } - + //If user wants 'ANY' road type acces then it is not included in query - + //returns every road type and null values if (accessRoadType !== "ANY") { query.accessRoadType = { [Op.eq]: accessRoadType diff --git a/app/helpers/db/searchRequest.js b/app/helpers/db/searchRequest.js index 808637a..e2633b4 100644 --- a/app/helpers/db/searchRequest.js +++ b/app/helpers/db/searchRequest.js @@ -49,128 +49,390 @@ const findSearchRequestsForRealEstate = async realEstate => { const geoSearchQueryPart = sequelize.where(contains, true); - //General query contains only attributes that are defined for every RealEstate - not null - const query = { - adType, - realEstateType, - subscribed: true, - [Op.and]: geoSearchQueryPart - }; //Needed for defining which attribute should exist or not const realEstateTypeObject = AD_CATEGORY[realEstateType]; - //Needed to decide on including incomplete RealEstates data + + // ?? Needed to decide on including incomplete RealEstates data let checkForIncompleteWanted = false; - //Attributes are checked separately and included in query only if defined - //Price and area should be defined for every property + //Attributes are checked separately to make different query parts - if (price) { - query.priceMin = { - [Op.lte]: price + //If real estate price is number then it searches for req that have priceMin and priceMax + //If real estate price is null it searches for req that accept ads without price + //User always defines price and area (sliders) - not null in search req + let priceQuery = {}; + if (price != null) { + priceQuery = { + [Op.and]: [ + { + priceMin: { + [Op.lte]: price + } + }, + { + priceMax: { + [Op.gte]: price + } + } + ] }; - query.priceMax = { - [Op.gte]: price + } else { + priceQuery = { + includeWithoutPrice: { + [Op.eq]: true + } }; } - if (area) { - query.sizeMin = { - [Op.lte]: area - }; - query.sizeMax = { - [Op.gte]: area + let areaQuery = {}; + if (area != null) { + areaQuery = { + [Op.and]: [ + { + sizeMin: { + [Op.lte]: area + } + }, + { + sizeMax: { + [Op.gte]: area + } + } + ] }; } else { checkForIncompleteWanted = true; } + //Other attributes can be defined or not depending on RealEstate type - if (gardenSize) { - query.gardenSizeMin = { - [Op.lte]: gardenSize - }; - query.gardenSizeMax = { - [Op.gte]: gardenSize - }; - } else if (realEstateTypeObject.hasGardenSize) { - checkForIncompleteWanted = true; + //we check what to include in query based on real estate type object + let gardenSizeQuery = {}; + if (realEstateTypeObject.hasGardenSize) { + if (gardenSize != null) { + gardenSizeQuery = { + [Op.and]: [ + { + gardenSizeMin: { + [Op.lte]: gardenSize + } + }, + { + gardenSizeMax: { + [Op.gte]: gardenSize + } + } + ] + }; + } else { + checkForIncompleteWanted = true; + } } - if (numberOfRooms) { - query.numberOfRoomsMin = { - [Op.lte]: numberOfRooms - }; - query.numberOfRoomsMax = { - [Op.gte]: numberOfRooms - }; - } else if (realEstateTypeObject.hasNumberOfRoom) { - checkForIncompleteWanted = true; + let numberOfRoomsQuery = {}; + if (realEstateTypeObject.hasNumberOfRoom) { + if (numberOfRooms != null) { + //If real estate has defined number of rooms ex. 3 it returns req + // that accepts 3 rooms or ones that don't have defined number - null + //Ex. they didnt choose advanced filters at all + numberOfRoomsQuery = { + [Op.and]: [ + { + numberOfRoomsMin: { + [Op.or]: { + [Op.lte]: numberOfRooms, + [Op.is]: null + } + } + }, + { + numberOfRoomsMax: { + [Op.or]: { + [Op.gte]: numberOfRooms, + [Op.is]: null + } + } + } + ] + }; + } else { + // If real estate dont have defined number of rooms ex. null + //It returns requests that didn't choose number of rooms - also null + //Or ones that picked some values but also picked to includeIncomplete ads + numberOfRoomsQuery = { + [Op.or]: [ + { + [Op.and]: [ + { + numberOfRoomsMin: { + [Op.is]: null + } + }, + { + numberOfRoomsMax: { + [Op.is]: null + } + } + ] + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + }; + } + } + //Same logic for number of Floors and floors + let numberOfFloorsQuery = {}; + if (realEstateTypeObject.hasNumberOfFloors) { + if (numberOfFloors != null) { + numberOfFloorsQuery = { + [Op.and]: [ + { + numberOfFloorsMin: { + [Op.or]: { + [Op.lte]: numberOfFloors, + [Op.is]: null + } + } + }, + { + numberOfFloorsMax: { + [Op.or]: { + [Op.gte]: numberOfFloors, + [Op.is]: null + } + } + } + ] + }; + } else { + numberOfFloorsQuery = { + [Op.or]: [ + { + [Op.and]: [ + { + numberOfFloorsMin: { + [Op.is]: null + } + }, + { + numberOfFloorsMax: { + [Op.is]: null + } + } + ] + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + }; + } + } + let floorQuery = {}; + if (realEstateTypeObject.hasFloorProp) { + if (floor != null) { + floorQuery = { + [Op.and]: [ + { + floorMin: { + [Op.or]: { + [Op.lte]: floor, + [Op.is]: null + } + } + }, + { + floorMax: { + [Op.or]: { + [Op.gte]: floor, + [Op.is]: null + } + } + } + ] + }; + } else { + floorQuery = { + [Op.or]: [ + { + [Op.and]: [ + { + floorMin: { + [Op.is]: null + } + }, + { + floorMax: { + [Op.is]: null + } + } + ] + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + }; + } } - if (numberOfFloors) { - query.numberOfFloorsMin = { - [Op.lte]: numberOfFloors - }; - query.numberOfFloorsMax = { - [Op.gte]: numberOfFloors - }; - } else if (realEstateTypeObject.hasNumberOfFloors) { - checkForIncompleteWanted = true; + //Logic for balcony, newBuilding and elevator + //If user dont check checkbox for ex. elevator it does not mean he only wants no elevator + //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query + //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true + //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads + let balconyQuery = {}; + if (realEstateTypeObject.hasBalconyProp && balcony !== true) { + if (balcony === false) { + balconyQuery = { + balcony: { + [Op.ne]: true + } + }; + } else if (balcony === null) { + balconyQuery = { + [Op.or]: [ + { + balcony: { + [Op.ne]: true + } + }, + { + [Op.and]: [ + { + balcony: { + [Op.eq]: true + } + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + } + ] + }; + } } - - if (floor) { - query.floorMin = { - [Op.lte]: floor - }; - query.floorMax = { - [Op.gte]: floor - }; - } else if (realEstateTypeObject.hasFloorProp) { - checkForIncompleteWanted = true; + let newBuildingQuery = {}; + if (realEstateTypeObject.hasNewBuildingProp && newBuilding !== true) { + if (newBuilding === false) { + newBuildingQuery = { + newBuilding: { + [Op.ne]: true + } + }; + } else if (newBuilding === null) { + newBuildingQuery = { + [Op.or]: [ + { + newBuilding: { + [Op.ne]: true + } + }, + { + [Op.and]: [ + { + newBuilding: { + [Op.eq]: true + } + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + } + ] + }; + } } + let elevatorQuery = {}; + if (realEstateTypeObject.hasElevatorProp && elevator !== true) { + if (elevator === false) { + elevatorQuery = { + elevator: { + [Op.ne]: true + } + }; + } else if (elevator === null) { + elevatorQuery = { + [Op.or]: [ + { + elevator: { + [Op.ne]: true + } + }, + { + [Op.and]: [ + { + elevator: { + [Op.eq]: true + } + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + } + ] + }; + } + } + //General query consists of each individual query + const query = { + adType, + realEstateType, + subscribed: true, + [Op.and]: [ + geoSearchQueryPart, + priceQuery, + areaQuery, + gardenSizeQuery, + numberOfRoomsQuery, + numberOfFloorsQuery, + floorQuery, + balconyQuery, + newBuildingQuery, + elevatorQuery + ] + }; - if (accessRoadType) { + //AccessRoadType is defined - should exists for each ad and estate type + if (accessRoadType != null) { query.accessRoadType = { [Op.or]: { - [Op.eq]: "ANY", + [Op.like]: "ANY", [Op.eq]: accessRoadType } }; - } else if (realEstateTypeObject.hasAccesRoadType) { - checkForIncompleteWanted = true; - } - - if (balcony) { - query.balcony = { - [Op.eq]: balcony + } else { + //Null values are returned for user request that wanted ANY acces road type + query.accessRoadType = { + [Op.eq]: "ANY" }; - } else if (realEstateTypeObject.hasBalconyProp) { - checkForIncompleteWanted = true; } - - if (newBuilding) { - query.newBuilding = { - [Op.eq]: newBuilding - }; - } else if (realEstateTypeObject.hasNewBuildingProp) { - checkForIncompleteWanted = true; - } - - if (elevator) { - query.elevator = { - [Op.eq]: elevator - }; - } else if (realEstateTypeObject.hasElevatorProp) { - checkForIncompleteWanted = true; - } - - //If one of the attributes that exists for property type is null - //we include in query to check if incomplete real estates are accepted + //Tag to check if incomplete ads are accepted in query if (checkForIncompleteWanted) { query.includeIncompleteAds = { [Op.eq]: true }; } - return await db.SearchRequest.findAll({ where: query }); + + return await db.SearchRequest.findAll({ + where: query + }); }; module.exports = { diff --git a/app/helpers/db/searchRequestMatch.js b/app/helpers/db/searchRequestMatch.js index 11d5cde..794711b 100644 --- a/app/helpers/db/searchRequestMatch.js +++ b/app/helpers/db/searchRequestMatch.js @@ -1,5 +1,8 @@ "use strict"; const db = require("../../models/index"); +const sequelize = require("sequelize"); +const Op = sequelize.Op; +const { CHECK_UP_DAYS } = require("../../config/appConfig"); const findRealEstatesForSearchRequest = async searchRequestId => { const query = { @@ -40,6 +43,42 @@ const findNotNotifiedMatches = async () => { return matchingRecords; }; +const findAllRequestsForCheckUp = async () => { + //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE + //The ones that received notification for real estate CHECK_UP_DAYS days from now + const date = new Date(); + const checkUpDate = date.getDate() - CHECK_UP_DAYS; + date.setDate(checkUpDate); + const dateQuery = { + createdAt: { + [Op.gte]: date + } + }; + + const excludedMatches = await db.SearchRequestMatch.findAll({ + attributes: ["searchRequestId"], + where: dateQuery, + order: [["searchRequestId", "ASC"]] + }); + + const excludedRequestsAll = excludedMatches.map(match => { + return match.dataValues.searchRequestId; + }); + //Removing duplicate search request id-s for optimization + const excludedRequests = [...new Set(excludedRequestsAll)]; + + const query = { + subscribed: true, + id: { + [Op.notIn]: excludedRequests + } + }; + const allRequestsForCheckUp = await db.SearchRequest.findAll({ + where: query + }); + + return allRequestsForCheckUp; +}; const addMatches = async matchingRecords => { return await db.SearchRequestMatch.bulkCreate(matchingRecords, { @@ -50,5 +89,6 @@ const addMatches = async matchingRecords => { module.exports = { findRealEstatesForSearchRequest, addMatches, - findNotNotifiedMatches + findNotNotifiedMatches, + findAllRequestsForCheckUp }; diff --git a/app/helpers/emailContentGenerator.js b/app/helpers/emailContentGenerator.js index e9efbfc..253d518 100644 --- a/app/helpers/emailContentGenerator.js +++ b/app/helpers/emailContentGenerator.js @@ -159,8 +159,42 @@ const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => { return `${stagingTag}Kivi: ${numberOfRealEstates} novih nekretnina`; }; +const generateCheckUpEmail = searchRequest => { + const realEstateType = AD_CATEGORY[searchRequest.realEstateType]; + const { + id, + gardenSizeMin, + gardenSizeMax, + sizeMin, + sizeMax, + priceMin, + priceMax + } = searchRequest; + + const gardenSize = realEstateType.hasGardenSize + ? `
Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2
` + : ``; + + const emailFooter = generateEmailFooter(id); + + return `

Zdravo

+
Kivi tim traži nekretnine za Vas i kada to ne vidite.
+
+
Vaša trenutno aktivna pretraga je:
+
+
+
Tip nekretnine: ${realEstateType.title}
+
Kvadratura nekretnine: Od ${sizeMin} do ${sizeMax} m2
+ ${gardenSize} +
Cijena: ${priceMin} do ${priceMax} KM
+
+
+ ${emailFooter}`; +}; + module.exports = { generateNotificationEmail, generateNewSearchRequestEmail, - generateEmailSubject + generateEmailSubject, + generateCheckUpEmail }; diff --git a/app/migrations/20200121000524-add-priceHistory-table.js b/app/migrations/20200121000524-add-priceHistory-table.js new file mode 100644 index 0000000..6d56f99 --- /dev/null +++ b/app/migrations/20200121000524-add-priceHistory-table.js @@ -0,0 +1,42 @@ +"use strict"; + +module.exports = { + up: (queryInterface, Sequelize) => { + const tableFields = { + id: { + type: Sequelize.BIGINT, + autoIncrement: true, + allowNull: false, + primaryKey: true + }, + realEstateId: { + type: Sequelize.BIGINT, + allowNull: false, + unique: "uniquePriceRealEstate", + references: { + model: "RealEstates", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: { + type: Sequelize.REAL, + unique: "uniquePriceRealEstate" + }, + createdAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.literal("NOW()") + }, + updatedAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.literal("NOW()") + } + }; + return queryInterface.createTable("PriceHistory", tableFields); + }, + + down: queryInterface => { + return queryInterface.dropTable("PriceHistory", {}); + } +}; diff --git a/app/migrations/20200121094500-add-constraint-priceHistory.js b/app/migrations/20200121094500-add-constraint-priceHistory.js new file mode 100644 index 0000000..45c83c3 --- /dev/null +++ b/app/migrations/20200121094500-add-constraint-priceHistory.js @@ -0,0 +1,10 @@ +"use strict"; +module.exports = { + up: (queryInterface, Sequelize) => + queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], { + type: "unique", + name: "uniquePriceRealEstate" + }), + down: queryInterface => + queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate") +}; diff --git a/app/migrations/20200123085754-add-column-includeWithoutPrice-to-searchRequests-table.js b/app/migrations/20200123085754-add-column-includeWithoutPrice-to-searchRequests-table.js new file mode 100644 index 0000000..9173829 --- /dev/null +++ b/app/migrations/20200123085754-add-column-includeWithoutPrice-to-searchRequests-table.js @@ -0,0 +1,14 @@ +"use strict"; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn("SearchRequests", "includeWithoutPrice", { + type: Sequelize.BOOLEAN, + defaultValue: true + }); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.removeColumn("SearchRequests", "includeWithoutPrice"); + } +}; diff --git a/app/models/priceHistory.js b/app/models/priceHistory.js new file mode 100644 index 0000000..495be53 --- /dev/null +++ b/app/models/priceHistory.js @@ -0,0 +1,44 @@ +"use strict"; + +module.exports = (sequalize, DataTypes) => { + const PriceHistory = sequalize.define( + "PriceHistory", + { + id: { + type: DataTypes.BIGINT, + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + realEstateId: { + type: DataTypes.BIGINT, + allowNull: false, + unique: "uniquePriceRealEstate", + references: { + model: "RealEstates", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: { + type: DataTypes.REAL, + unique: "uniquePriceRealEstate" + } + }, + { + freezeTableName: true + } + ); + + PriceHistory.associate = models => { + PriceHistory.hasMany(models.RealEstate, { + foreignKey: "id", + sourceKey: "realEstateId", + targetKey: "id", + as: "realEstates" + }); + }; + + return PriceHistory; +}; diff --git a/app/models/searchRequest.js b/app/models/searchRequest.js index 8a04593..0d4997f 100644 --- a/app/models/searchRequest.js +++ b/app/models/searchRequest.js @@ -15,7 +15,15 @@ module.exports = (sequelize, DataTypes) => { allowNull: false, defaultValue: { type: "Polygon", - coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]], + coordinates: [ + [ + [0, 0], + [0, 0], + [0, 0], + [0, 0], + [0, 0] + ] + ], crs: { type: "name", properties: { name: "EPSG:4326" } } } }, @@ -71,6 +79,7 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.TEXT }, includeIncompleteAds: DataTypes.BOOLEAN, + includeWithoutPrice: DataTypes.BOOLEAN, balcony: DataTypes.BOOLEAN, elevator: DataTypes.BOOLEAN, newBuilding: DataTypes.BOOLEAN, diff --git a/app/npmScripts/npmCheckUpNotify.js b/app/npmScripts/npmCheckUpNotify.js new file mode 100644 index 0000000..b69cbf6 --- /dev/null +++ b/app/npmScripts/npmCheckUpNotify.js @@ -0,0 +1,6 @@ +"use strict"; +const { checkUpNotify } = require("../services/notificationService"); +//For testing pursposes +(async () => { + await checkUpNotify(); +})(); diff --git a/app/seeders/20200121150443-initial-price-history.js b/app/seeders/20200121150443-initial-price-history.js new file mode 100644 index 0000000..0a9531a --- /dev/null +++ b/app/seeders/20200121150443-initial-price-history.js @@ -0,0 +1,34 @@ +"use strict"; + +const { RealEstate } = require("../models"); + +module.exports = { + async up(queryInterface, Sequelize) { + //Reading initial data from RealEstate table in db + const realEstateInitialData = await RealEstate.findAll(); + //Extruding data for table PriceHistory + const priceHistoryInitialData = realEstateInitialData.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + + return { + realEstateId: realEstate.dataValues.id, + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt + }; + }); + + return queryInterface.bulkInsert( + "PriceHistory", + priceHistoryInitialData, + {} + ); + }, + + async down(queryInterface, Sequelize) { + return queryInterface.bulkDelete("PriceHistory", null, {}); + } +}; diff --git a/app/services/notificationService.js b/app/services/notificationService.js index 1d05283..1e05e5e 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -7,10 +7,12 @@ const { const { generateNotificationEmail, generateNewSearchRequestEmail, - generateEmailSubject + generateEmailSubject, + generateCheckUpEmail } = require("../helpers/emailContentGenerator"); const { findNotNotifiedMatches, + findAllRequestsForCheckUp, findRealEstatesForSearchRequest } = require("../helpers/db/searchRequestMatch"); const { sendEmail } = require("../services/emailService"); @@ -125,8 +127,26 @@ const notifyRequestsWithDailyOption = async () => { await notifyMatches(matches, true); }; +const checkUpNotify = async () => { + const searchRequestsForCheckUp = await findAllRequestsForCheckUp(); + + const asyncSendEmailActions = []; + + for (const searchRequest of searchRequestsForCheckUp) { + const { email } = searchRequest.dataValues; + const emailSubject = `Kivi: Mi tražimo nekretnine za vas!`; + const emailContent = generateCheckUpEmail(searchRequest.dataValues); + + const sendEmailPromise = sendEmail(email, emailSubject, emailContent); + asyncSendEmailActions.push(sendEmailPromise); + sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err)); + } + await Promise.all(asyncSendEmailActions); +}; + module.exports = { notifyForNewRealEstates, notifyForNewSearchRequest, - notifyRequestsWithDailyOption + notifyRequestsWithDailyOption, + checkUpNotify }; diff --git a/app/views/standardFilters.ejs b/app/views/standardFilters.ejs index 27a6bba..43b931a 100644 --- a/app/views/standardFilters.ejs +++ b/app/views/standardFilters.ejs @@ -18,6 +18,15 @@ +
+

+ +


diff --git a/development.env b/development.env index 4bfb37d..27b54e1 100644 --- a/development.env +++ b/development.env @@ -14,6 +14,7 @@ SETTINGS=Variable to denote development, staging and production MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email +CHECK_UP_DAYS=Check up email is sent after this number of days without notification #=============== GOOGLE ANALYTICS =============# GA_ID=Google Analytics ID diff --git a/index.js b/index.js index 074099b..0669e7b 100644 --- a/index.js +++ b/index.js @@ -12,6 +12,7 @@ const { } = require("./app/config/appConfig"); const routes = require("./app/routes"); const { crawlAll } = require("./app/crawler/crawl"); +const { checkUpNotify } = require("./app/services/notificationService"); const { notifyForNewRealEstates } = require("./app/services/notificationService"); @@ -45,4 +46,7 @@ const crawl = () => { }); } }; + setInterval(crawl, CRAWLER_INTERVAL * 1000); + +setInterval(checkUpNotify, 1000 * 60 * 60 * 24); diff --git a/package.json b/package.json index 511f772..3bc99a7 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,13 @@ "start": "node ./index.js", "start-mon": "nodemon ./index.js", "migrate": "cd app && npx sequelize db:migrate", + "seed": "cd app && npx sequelize db:seed:all", "setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 10 && npm run migrate", "docker-start": "docker start pg_marketalerts", "docker-stop": "docker stop pg_marketalerts", "crawl": "cd app/crawler && node npmCrawl.js", "daily-notify": "cd app/npmScripts && node npmDailyNotify.js", + "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js", "test-search": "cd test && node searchTest.js", "test-olx-scraper": "cd test && node olxScrapeTest.js", "test-rental-scraper": "cd test && node rentalScrapeTest.js"