diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..82435b1 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +*.ejs \ No newline at end of file diff --git a/app/common/enums.js b/app/common/enums.js index 33cb41e..20aa4dd 100644 --- a/app/common/enums.js +++ b/app/common/enums.js @@ -7,7 +7,42 @@ const PRICE_SLIDER_OPTIONS_SALE = { step: 1000, connect: true }; - +const FLAT_PRICE_SLIDER_OPTIONS_SALE = { + start: [50000, 150000], + range: { + min: [0], + max: [800000] + }, + step: 5000, + connect: true +}; +const HOUSE_PRICE_SLIDER_OPTIONS_SALE = { + start: [50000, 150000], + range: { + min: [0], + max: [1500000] + }, + step: 10000, + connect: true +}; +const OFFICE_PRICE_SLIDER_OPTIONS_SALE = { + start: [15000, 50000], + range: { + min: [0], + max: [2000000] + }, + step: 2000, + connect: true +}; +const LAND_PRICE_SLIDER_OPTIONS_SALE = { + start: [40000, 80000], + range: { + min: [0], + max: [2000000] + }, + step: 10000, + connect: true +}; const PRICE_SLIDER_OPTIONS_RENT = { start: [300, 500], range: { @@ -17,18 +52,62 @@ const PRICE_SLIDER_OPTIONS_RENT = { step: 50, connect: true }; - +const FLAT_PRICE_SLIDER_OPTIONS_RENT = { + start: [300, 600], + range: { + min: [0], + max: [4000] + }, + step: 100, + connect: true +}; +const HOUSE_PRICE_SLIDER_OPTIONS_RENT = { + start: [500, 1000], + range: { + min: [0], + max: [10000] + }, + step: 100, + connect: true +}; +const OFFICE_PRICE_SLIDER_OPTIONS_RENT = { + start: [200, 1000], + range: { + min: [0], + max: [20000] + }, + step: 100, + connect: true +}; +const LAND_PRICE_SLIDER_OPTIONS_RENT = { + start: [500, 1000], + range: { + min: [0], + max: [20000] + }, + step: 100, + connect: true +}; //This will be used for Flats, Apartments, Houses const HOME_SIZE_SLIDER_OPTIONS = { start: [30, 75], range: { min: [0], - max: [400] + max: [500] }, step: 5, connect: true }; +const OFFICE_SIZE_SLIDER_OPTIONS = { + start: [30, 150], + range: { + min: [0], + max: [1200] + }, + step: 10, + connect: true +}; const GARDEN_SIZE_SLIDER_OPTIONS = { start: [100, 1000], range: { @@ -111,8 +190,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: false, hasFloorProp: true, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS }, HOUSE: { @@ -126,8 +205,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: true, hasFloorProp: false, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS, gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS }, @@ -142,9 +221,9 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: false, hasFloorProp: true, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, - sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS + priceSliderOptionsSale: OFFICE_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: OFFICE_PRICE_SLIDER_OPTIONS_RENT, + sizeSliderOptions: OFFICE_SIZE_SLIDER_OPTIONS }, LAND: { id: "LAND", @@ -157,8 +236,8 @@ const AD_CATEGORY = { hasNumberOfRoom: false, hasNumberOfFloors: false, hasFloorProp: false, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: LAND_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: LAND_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS }, APARTMENT: { @@ -172,8 +251,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: false, hasFloorProp: true, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS }, GARAGE: { @@ -202,8 +281,8 @@ const AD_CATEGORY = { hasNumberOfRoom: true, hasNumberOfFloors: true, hasFloorProp: false, - priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE, - priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT, + priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE, + priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT, sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS, gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS } @@ -216,7 +295,8 @@ const AD_STATUS = { STATUS_DELETED: 4, STATUS_URGENT: 5, STATUS_DISCOUNTED: 6, - STATUS_RENTED: 7 + STATUS_RENTED: 7, + STATUS_VIP: 8 }; const AD_AGENCY = { diff --git a/app/config/appConfig.js b/app/config/appConfig.js index d269e6f..3ccedbe 100644 --- a/app/config/appConfig.js +++ b/app/config/appConfig.js @@ -34,6 +34,11 @@ const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0; const API_MAP_KEY = process.env.API_MAP_KEY || ""; +const PROSTOR_LOGIN = { + EMAIL: process.env.PROSTOR_LOGIN_EMAIL, + PASSWORD: process.env.PROSTOR_LOGIN_PASS +}; + module.exports = { APP_PORT, APP_URL, @@ -45,5 +50,6 @@ module.exports = { MAX_REAL_ESTATES_IN_FIRST_EMAIL, PRINT_CRAWLER_DEBUG, API_MAP_KEY, - CHECK_UP_DAYS + CHECK_UP_DAYS, + PROSTOR_LOGIN }; diff --git a/app/controllers/realEstateFilters.js b/app/controllers/realEstateFilters.js index 17f5e58..252ac46 100644 --- a/app/controllers/realEstateFilters.js +++ b/app/controllers/realEstateFilters.js @@ -35,7 +35,8 @@ const getFilters = async (req, res) => { balcony, elevator, newBuilding, - accessRoadType + accessRoadType, + includeWithoutPrice } = searchRequest; const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT; @@ -115,7 +116,8 @@ const getFilters = async (req, res) => { advancedSegmentSelectFilterValues, advancedRangeFilterObjects, advancedRangeFilterValues, - includeIncompleteAds + includeIncompleteAds, + includeWithoutPrice }); }; @@ -191,6 +193,7 @@ const postFilters = async (req, res) => { }); const includeIncompleteAds = req.body.includeIncompleteAds === "on"; + const includeWithoutPrice = req.body.includeWithoutPrice === "on"; const balcony = req.body.balcony === "on"; const elevator = req.body.elevator === "on"; @@ -217,6 +220,7 @@ const postFilters = async (req, res) => { searchRequest.newBuilding = newBuilding; searchRequest.includeIncompleteAds = includeIncompleteAds; + searchRequest.includeWithoutPrice = includeWithoutPrice; searchRequest.accessRoadType = accessRoadType; diff --git a/app/controllers/realEstates.js b/app/controllers/realEstates.js index ce82765..48c1aff 100644 --- a/app/controllers/realEstates.js +++ b/app/controllers/realEstates.js @@ -2,13 +2,14 @@ const { findRealEstatesForSearchRequest } = require("../helpers/db/searchRequestMatch"); +const { AD_STATUS } = require("../common/enums"); const getRealEstates = async (req, res) => { const searchRequestId = req.params["searchRequestId"] || ""; const realEstates = await findRealEstatesForSearchRequest(searchRequestId); const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage"; - res.render("realEstates", { realEstates, title }); + res.render("realEstates", { realEstates, title, AD_STATUS }); }; module.exports = { diff --git a/app/controllers/redirect.js b/app/controllers/redirect.js index 9975ab2..eb4e505 100644 --- a/app/controllers/redirect.js +++ b/app/controllers/redirect.js @@ -1,9 +1,11 @@ const { getRealEstateById } = require("../helpers/db/realEstate"); +const { AD_STATUS } = require("../common/enums"); const getRedirect = async (req, res) => { const id = req.params.id || null; let error = false; let redirectUrl = undefined; + let vipAd = undefined; if (!id) { error = true; } else { @@ -13,6 +15,7 @@ const getRedirect = async (req, res) => { error = true; } else { redirectUrl = realEstate.url; + vipAd = realEstate.adStatus === AD_STATUS.STATUS_VIP; } } catch (e) { error = true; @@ -24,7 +27,7 @@ const getRedirect = async (req, res) => { res.render("notFound", { title }); } else { const title = "Preusmjeravanje"; - res.render("redirect", { title, redirectUrl }); + res.render("redirect", { title, redirectUrl, vipAd }); } }; diff --git a/app/crawler/savers/postgres.js b/app/crawler/savers/postgres.js index 344e4ac..97ba285 100644 --- a/app/crawler/savers/postgres.js +++ b/app/crawler/savers/postgres.js @@ -1,6 +1,7 @@ const moment = require("moment"); const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate"); +const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory"); class PostgresSaver { connect() { @@ -11,6 +12,21 @@ class PostgresSaver { async save(results) { const savedRecords = await bulkUpsertRealEstates(results); + //Extruding data for price history table + const resultPrices = savedRecords.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + + return { + realEstateId: realEstate.dataValues.id, + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt + }; + }); + const savedPrices = await bulkUpsertPriceHistory(resultPrices); if (Array.isArray(savedRecords)) { const newRealEstates = []; diff --git a/app/crawler/specificCrawlers/prostor.js b/app/crawler/specificCrawlers/prostor.js index cb1d3f9..04be5f3 100644 --- a/app/crawler/specificCrawlers/prostor.js +++ b/app/crawler/specificCrawlers/prostor.js @@ -3,6 +3,7 @@ const fetch = require("node-fetch"); const cheerio = require("cheerio"); const moment = require("moment-timezone"); +const FormData = require("form-data"); const { AD_TYPE, @@ -16,7 +17,8 @@ const { const { PRINT_CRAWLER_DEBUG, - DEFAULT_TIMEZONE + DEFAULT_TIMEZONE, + PROSTOR_LOGIN } = require("../../config/appConfig"); const { PROSTOR_FORCE_CRAWL } = require("../specificConfigs/prostor"); @@ -60,13 +62,16 @@ class ProstorCrawler { async crawl() { const crawlAdCategories = this.crawlerAdCategories; - + //We need session cookie to use login privileges + const prostorCookie = await this.getCookies(); + //New tag to check if crawler loged in + const login = await this.loginForScraping(PROSTOR_LOGIN, prostorCookie); const newRealEstates = []; - - if (crawlAdCategories) { + //Crawl only if login was successful + if (crawlAdCategories && login) { const indexGenerators = []; for (const adCategory of crawlAdCategories) { - indexGenerators.push(this.categoryIndexer(adCategory)); + indexGenerators.push(this.categoryIndexer(adCategory, prostorCookie)); } let done = false; @@ -119,13 +124,14 @@ class ProstorCrawler { return newRealEstates; } - async *categoryIndexer(adCategory) { + async *categoryIndexer(adCategory, prostorCookie) { const urlAdTypePart = PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes]; const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory]; if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) { const urlPageToCrawl = `${this.baseUrl}?remove_sold=0${urlAdTypePart}${urlCategoryPart}`; const listOfAllRealEstates = await this.extractRealEstates( - urlPageToCrawl + urlPageToCrawl, + prostorCookie ); let elementToStartIndexFrom = 0; @@ -139,7 +145,8 @@ class ProstorCrawler { elementToStartIndexFrom += realEstatesForSinglePage.length; const singlePageResults = await this.indexSinglePage( - realEstatesForSinglePage + realEstatesForSinglePage, + prostorCookie ); const filteredSinglePageResults = singlePageResults.filter( @@ -163,10 +170,10 @@ class ProstorCrawler { } } - async indexSinglePage(realEstatesList) { + async indexSinglePage(realEstatesList, prostorCookie) { const asyncActions = []; for (const realEstate of realEstatesList) { - asyncActions.push(this.scrapeAd(realEstate)); + asyncActions.push(this.scrapeAd(realEstate, prostorCookie)); } try { @@ -180,12 +187,25 @@ class ProstorCrawler { } } - async scrapeAd(realEstate) { + async scrapeAd(realEstate, prostorCookie) { const { lat, lng, property_name, price, size, link, status } = realEstate; + + //Status information is given already in realestate list + //For VIP Ads status ='' canot be used, but no VIP ads are crawled + //We will make "fake" vip ad for RE that have size=55 + //It is weird because yesterday it said 'VIP ponuda' ??? + const adStatus = + size === "55" + ? ProstorCrawler.getStatusId("VIP ponuda") + : ProstorCrawler.getStatusId(status); + const url = `https://prostor.ba${link}`; + // console.log("[PROSTOR] Scraping : ", url); try { - const adPageSource = await fetch(url); + const adPageSource = await fetch(url, { + headers: { Cookie: prostorCookie } + }); const body = await adPageSource.text(); const $ = cheerio.load(body); @@ -330,7 +350,6 @@ class ProstorCrawler { furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id; } - const adStatus = ProstorCrawler.getStatusId(status); const title = property_name; const parsedPrice = parseFloat(price.replace(/\./g, "")) || null; const parsedArea = parseFloat(size); @@ -408,13 +427,15 @@ class ProstorCrawler { } } - async extractRealEstates(url) { + async extractRealEstates(url, prostorCookie) { if (PRINT_CRAWLER_DEBUG) { console.log("[PROSTOR] Index page : ", url); } try { - const res = await fetch(url); + const res = await fetch(url, { + headers: { Cookie: prostorCookie } + }); const body = await res.text(); const $ = cheerio.load(body); @@ -548,6 +569,8 @@ class ProstorCrawler { return AD_STATUS.STATUS_SOLD; case "Iznajmljeno": return AD_STATUS.STATUS_RENTED; + case "VIP ponuda": + return AD_STATUS.STATUS_VIP; default: console.log("[PROSTOR] Unknown AD_STATUS : [", statusText, "]"); return AD_STATUS.STATUS_NORMAL; @@ -569,6 +592,51 @@ class ProstorCrawler { return savers[0].save(results); //so that we can use some sequelize options and information when data is inserted } + async loginForScraping(PROSTOR_LOGIN, prostorCookie) { + let formData = new FormData(); + formData.append("email", PROSTOR_LOGIN.EMAIL); + formData.append("password", PROSTOR_LOGIN.PASSWORD); + + return fetch("https://prostor.ba/moj-prostor/prijava", { + method: "POST", + body: formData, + headers: { Cookie: prostorCookie } + }) + .then(page => { + return page.text(); + }) + .then(resp => { + const $ = cheerio.load(resp); + if ( + $("h1") + .text() + .indexOf("Dobrodošli") !== -1 + ) { + console.log("[PROSTOR]: Crawler loged in!"); + return true; + } else { + console.log("[PROSTOR]: Crawler login failed - wrong credentials!"); + return false; + } + }) + .catch(err => { + console.log("[PROSTOR]: Crawler login error ", err); + }); + } + async getCookies() { + const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", { + headers: { Cookie: "" } + }); + const raw = getResponse.headers.raw()["set-cookie"]; + const cookie = raw + .map(datastring => { + const data = datastring.split(";"); + const cookieData = data[0]; + return cookieData; + }) + .join(";"); + return cookie; + } } module.exports = ProstorCrawler; diff --git a/app/helpers/db/priceHistory.js b/app/helpers/db/priceHistory.js new file mode 100644 index 0000000..3e8f69e --- /dev/null +++ b/app/helpers/db/priceHistory.js @@ -0,0 +1,20 @@ +"use strict"; +const db = require("../../models/index"); +const sequelize = require("sequelize"); + +const bulkUpsertPriceHistory = async priceHistoryData => { + try { + const order = [["realEstateId", "desc"]]; + + return await db.PriceHistory.bulkCreate(priceHistoryData, { + order, + ignoreDuplicates: true + }); + } catch (e) { + console.log("Error bulk upserting priceHistory : ", e); + } +}; + +module.exports = { + bulkUpsertPriceHistory +}; diff --git a/app/helpers/db/realEstate.js b/app/helpers/db/realEstate.js index ebeb84c..0f77260 100644 --- a/app/helpers/db/realEstate.js +++ b/app/helpers/db/realEstate.js @@ -2,6 +2,8 @@ const db = require("../../models/index"); const sequelize = require("sequelize"); const Op = sequelize.Op; +const { AD_CATEGORY } = require("../../common/enums"); + const bulkUpsertRealEstates = async realEstateData => { try { const fieldsToUpdateIfDuplicate = [ @@ -96,12 +98,16 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { floorMin, floorMax, includeIncompleteAds, + includeWithoutPrice, balcony, elevator, newBuilding, accessRoadType } = searchRequest; + //Needed for defining which attribute should exist or not + const realEstateTypeObject = AD_CATEGORY[realEstateType]; + const longitudeColumn = sequelize.col("locationLong"); const latitudeColumn = sequelize.col("locationLat"); @@ -134,15 +140,6 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { const query = { adType, realEstateType, - price: { - [Op.or]: { - [Op.and]: { - [Op.lte]: priceMax, - [Op.gte]: priceMin - }, - [Op.is]: null - } - }, area: { [Op.lte]: sizeMax, [Op.gte]: sizeMin @@ -154,15 +151,6 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { const queryIncludeIncomplete = { adType, realEstateType, - price: { - [Op.or]: { - [Op.and]: { - [Op.lte]: priceMax, - [Op.gte]: priceMin - }, - [Op.is]: null - } - }, area: { [Op.or]: { [Op.and]: { @@ -175,8 +163,49 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { [Op.and]: geoSearchQueryPart }; - //Every other attribute is checked separately and included in query only if it is defined - if (gardenSizeMax && gardenSizeMin) { + //Is user unchecked includeWithoutPrice FALSE then it shouldn't return null values of price + //If not then null values are accepted (this is DEFAULT) + //includeIncpompleteAds does not have effect on price query + if (includeWithoutPrice) { + query.price = { + [Op.or]: { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + }, + [Op.is]: null + } + }; + queryIncludeIncomplete.price = { + [Op.or]: { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + }, + [Op.is]: null + } + }; + } else { + query.price = { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + } + }; + queryIncludeIncomplete.price = { + [Op.and]: { + [Op.lte]: priceMax, + [Op.gte]: priceMin + } + }; + } + //Every other attribute is checked separately and included in query only if it is defined for real estate type + + if ( + realEstateTypeObject.hasGardenSize && + gardenSizeMax != null && + gardenSizeMin != null + ) { query.gardenSize = { [Op.lte]: gardenSizeMax, [Op.gte]: gardenSizeMin @@ -192,7 +221,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (numberOfRoomsMin && numberOfRoomsMax) { + if ( + realEstateTypeObject.hasNumberOfRoom && + numberOfRoomsMin != null && + numberOfRoomsMax != null + ) { query.numberOfRooms = { [Op.lte]: numberOfRoomsMax, [Op.gte]: numberOfRoomsMin @@ -208,7 +241,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (numberOfFloorsMin && numberOfFloorsMax) { + if ( + realEstateTypeObject.hasNumberOfFloors && + numberOfFloorsMin != null && + numberOfFloorsMax != null + ) { query.numberOfFloors = { [Op.lte]: numberOfFloorsMax, [Op.gte]: numberOfFloorsMin @@ -224,7 +261,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (floorMin && floorMax) { + if ( + realEstateTypeObject.hasFloorProp && + floorMin != null && + floorMax != null + ) { query.floor = { [Op.lte]: floorMax, [Op.gte]: floorMin @@ -239,8 +280,10 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { } }; } - - if (balcony) { + //Logic for balcony, newBuilding and elevator from users side + //If true is checked, then I want characteristic to be true but, + //if it is not checked, then I dont care - it can be null or false or true + if (realEstateTypeObject.hasBalconyProp && balcony === true) { query.balcony = { [Op.eq]: balcony }; @@ -252,7 +295,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (newBuilding) { + if (realEstateTypeObject.hasNewBuildingProp && newBuilding === true) { query.newBuilding = { [Op.eq]: newBuilding }; @@ -264,7 +307,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { }; } - if (elevator) { + if (realEstateTypeObject.hasElevatorProp && elevator === true) { query.elevator = { [Op.eq]: elevator }; @@ -275,7 +318,8 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => { } }; } - + //If user wants 'ANY' road type acces then it is not included in query - + //returns every road type and null values if (accessRoadType !== "ANY") { query.accessRoadType = { [Op.eq]: accessRoadType diff --git a/app/helpers/db/searchRequest.js b/app/helpers/db/searchRequest.js index 808637a..e2633b4 100644 --- a/app/helpers/db/searchRequest.js +++ b/app/helpers/db/searchRequest.js @@ -49,128 +49,390 @@ const findSearchRequestsForRealEstate = async realEstate => { const geoSearchQueryPart = sequelize.where(contains, true); - //General query contains only attributes that are defined for every RealEstate - not null - const query = { - adType, - realEstateType, - subscribed: true, - [Op.and]: geoSearchQueryPart - }; //Needed for defining which attribute should exist or not const realEstateTypeObject = AD_CATEGORY[realEstateType]; - //Needed to decide on including incomplete RealEstates data + + // ?? Needed to decide on including incomplete RealEstates data let checkForIncompleteWanted = false; - //Attributes are checked separately and included in query only if defined - //Price and area should be defined for every property + //Attributes are checked separately to make different query parts - if (price) { - query.priceMin = { - [Op.lte]: price + //If real estate price is number then it searches for req that have priceMin and priceMax + //If real estate price is null it searches for req that accept ads without price + //User always defines price and area (sliders) - not null in search req + let priceQuery = {}; + if (price != null) { + priceQuery = { + [Op.and]: [ + { + priceMin: { + [Op.lte]: price + } + }, + { + priceMax: { + [Op.gte]: price + } + } + ] }; - query.priceMax = { - [Op.gte]: price + } else { + priceQuery = { + includeWithoutPrice: { + [Op.eq]: true + } }; } - if (area) { - query.sizeMin = { - [Op.lte]: area - }; - query.sizeMax = { - [Op.gte]: area + let areaQuery = {}; + if (area != null) { + areaQuery = { + [Op.and]: [ + { + sizeMin: { + [Op.lte]: area + } + }, + { + sizeMax: { + [Op.gte]: area + } + } + ] }; } else { checkForIncompleteWanted = true; } + //Other attributes can be defined or not depending on RealEstate type - if (gardenSize) { - query.gardenSizeMin = { - [Op.lte]: gardenSize - }; - query.gardenSizeMax = { - [Op.gte]: gardenSize - }; - } else if (realEstateTypeObject.hasGardenSize) { - checkForIncompleteWanted = true; + //we check what to include in query based on real estate type object + let gardenSizeQuery = {}; + if (realEstateTypeObject.hasGardenSize) { + if (gardenSize != null) { + gardenSizeQuery = { + [Op.and]: [ + { + gardenSizeMin: { + [Op.lte]: gardenSize + } + }, + { + gardenSizeMax: { + [Op.gte]: gardenSize + } + } + ] + }; + } else { + checkForIncompleteWanted = true; + } } - if (numberOfRooms) { - query.numberOfRoomsMin = { - [Op.lte]: numberOfRooms - }; - query.numberOfRoomsMax = { - [Op.gte]: numberOfRooms - }; - } else if (realEstateTypeObject.hasNumberOfRoom) { - checkForIncompleteWanted = true; + let numberOfRoomsQuery = {}; + if (realEstateTypeObject.hasNumberOfRoom) { + if (numberOfRooms != null) { + //If real estate has defined number of rooms ex. 3 it returns req + // that accepts 3 rooms or ones that don't have defined number - null + //Ex. they didnt choose advanced filters at all + numberOfRoomsQuery = { + [Op.and]: [ + { + numberOfRoomsMin: { + [Op.or]: { + [Op.lte]: numberOfRooms, + [Op.is]: null + } + } + }, + { + numberOfRoomsMax: { + [Op.or]: { + [Op.gte]: numberOfRooms, + [Op.is]: null + } + } + } + ] + }; + } else { + // If real estate dont have defined number of rooms ex. null + //It returns requests that didn't choose number of rooms - also null + //Or ones that picked some values but also picked to includeIncomplete ads + numberOfRoomsQuery = { + [Op.or]: [ + { + [Op.and]: [ + { + numberOfRoomsMin: { + [Op.is]: null + } + }, + { + numberOfRoomsMax: { + [Op.is]: null + } + } + ] + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + }; + } + } + //Same logic for number of Floors and floors + let numberOfFloorsQuery = {}; + if (realEstateTypeObject.hasNumberOfFloors) { + if (numberOfFloors != null) { + numberOfFloorsQuery = { + [Op.and]: [ + { + numberOfFloorsMin: { + [Op.or]: { + [Op.lte]: numberOfFloors, + [Op.is]: null + } + } + }, + { + numberOfFloorsMax: { + [Op.or]: { + [Op.gte]: numberOfFloors, + [Op.is]: null + } + } + } + ] + }; + } else { + numberOfFloorsQuery = { + [Op.or]: [ + { + [Op.and]: [ + { + numberOfFloorsMin: { + [Op.is]: null + } + }, + { + numberOfFloorsMax: { + [Op.is]: null + } + } + ] + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + }; + } + } + let floorQuery = {}; + if (realEstateTypeObject.hasFloorProp) { + if (floor != null) { + floorQuery = { + [Op.and]: [ + { + floorMin: { + [Op.or]: { + [Op.lte]: floor, + [Op.is]: null + } + } + }, + { + floorMax: { + [Op.or]: { + [Op.gte]: floor, + [Op.is]: null + } + } + } + ] + }; + } else { + floorQuery = { + [Op.or]: [ + { + [Op.and]: [ + { + floorMin: { + [Op.is]: null + } + }, + { + floorMax: { + [Op.is]: null + } + } + ] + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + }; + } } - if (numberOfFloors) { - query.numberOfFloorsMin = { - [Op.lte]: numberOfFloors - }; - query.numberOfFloorsMax = { - [Op.gte]: numberOfFloors - }; - } else if (realEstateTypeObject.hasNumberOfFloors) { - checkForIncompleteWanted = true; + //Logic for balcony, newBuilding and elevator + //If user dont check checkbox for ex. elevator it does not mean he only wants no elevator + //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query + //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true + //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads + let balconyQuery = {}; + if (realEstateTypeObject.hasBalconyProp && balcony !== true) { + if (balcony === false) { + balconyQuery = { + balcony: { + [Op.ne]: true + } + }; + } else if (balcony === null) { + balconyQuery = { + [Op.or]: [ + { + balcony: { + [Op.ne]: true + } + }, + { + [Op.and]: [ + { + balcony: { + [Op.eq]: true + } + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + } + ] + }; + } } - - if (floor) { - query.floorMin = { - [Op.lte]: floor - }; - query.floorMax = { - [Op.gte]: floor - }; - } else if (realEstateTypeObject.hasFloorProp) { - checkForIncompleteWanted = true; + let newBuildingQuery = {}; + if (realEstateTypeObject.hasNewBuildingProp && newBuilding !== true) { + if (newBuilding === false) { + newBuildingQuery = { + newBuilding: { + [Op.ne]: true + } + }; + } else if (newBuilding === null) { + newBuildingQuery = { + [Op.or]: [ + { + newBuilding: { + [Op.ne]: true + } + }, + { + [Op.and]: [ + { + newBuilding: { + [Op.eq]: true + } + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + } + ] + }; + } } + let elevatorQuery = {}; + if (realEstateTypeObject.hasElevatorProp && elevator !== true) { + if (elevator === false) { + elevatorQuery = { + elevator: { + [Op.ne]: true + } + }; + } else if (elevator === null) { + elevatorQuery = { + [Op.or]: [ + { + elevator: { + [Op.ne]: true + } + }, + { + [Op.and]: [ + { + elevator: { + [Op.eq]: true + } + }, + { + includeIncompleteAds: { + [Op.eq]: true + } + } + ] + } + ] + }; + } + } + //General query consists of each individual query + const query = { + adType, + realEstateType, + subscribed: true, + [Op.and]: [ + geoSearchQueryPart, + priceQuery, + areaQuery, + gardenSizeQuery, + numberOfRoomsQuery, + numberOfFloorsQuery, + floorQuery, + balconyQuery, + newBuildingQuery, + elevatorQuery + ] + }; - if (accessRoadType) { + //AccessRoadType is defined - should exists for each ad and estate type + if (accessRoadType != null) { query.accessRoadType = { [Op.or]: { - [Op.eq]: "ANY", + [Op.like]: "ANY", [Op.eq]: accessRoadType } }; - } else if (realEstateTypeObject.hasAccesRoadType) { - checkForIncompleteWanted = true; - } - - if (balcony) { - query.balcony = { - [Op.eq]: balcony + } else { + //Null values are returned for user request that wanted ANY acces road type + query.accessRoadType = { + [Op.eq]: "ANY" }; - } else if (realEstateTypeObject.hasBalconyProp) { - checkForIncompleteWanted = true; } - - if (newBuilding) { - query.newBuilding = { - [Op.eq]: newBuilding - }; - } else if (realEstateTypeObject.hasNewBuildingProp) { - checkForIncompleteWanted = true; - } - - if (elevator) { - query.elevator = { - [Op.eq]: elevator - }; - } else if (realEstateTypeObject.hasElevatorProp) { - checkForIncompleteWanted = true; - } - - //If one of the attributes that exists for property type is null - //we include in query to check if incomplete real estates are accepted + //Tag to check if incomplete ads are accepted in query if (checkForIncompleteWanted) { query.includeIncompleteAds = { [Op.eq]: true }; } - return await db.SearchRequest.findAll({ where: query }); + + return await db.SearchRequest.findAll({ + where: query + }); }; module.exports = { diff --git a/app/helpers/emailContentGenerator.js b/app/helpers/emailContentGenerator.js index 4bd2c59..5a76e9b 100644 --- a/app/helpers/emailContentGenerator.js +++ b/app/helpers/emailContentGenerator.js @@ -1,10 +1,11 @@ "use strict"; const { MAX_REAL_ESTATES_IN_EMAIL, APP_URL } = require("../config/appConfig"); -const { AD_CATEGORY } = require("../common/enums"); +const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums"); -const generateEmailFooter = searchRequestId => { - return `
+ +