From ef5de27c067cabfa24080a0d54bdc379c46987ad Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Wed, 18 Dec 2019 22:21:56 +0100 Subject: [PATCH 1/5] Add currency to price filters - added above input --- app/views/standardFilters.ejs | 67 +++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/app/views/standardFilters.ejs b/app/views/standardFilters.ejs index 423d694..27a6bba 100644 --- a/app/views/standardFilters.ejs +++ b/app/views/standardFilters.ejs @@ -1,66 +1,73 @@ -
+
-
Cijena
-

+
Cijena (KM)
+

-
+
- - +
- +
-
+
-
Površina
-

+
Površina (m2)
+

-
+
- - +
- +
-
+
<% if(hasGardenSize) { %> -
-
Površina okućnice
-

-
-
-
+
+
Površina okućnice (m2)
+

+
+
+
-
-
-
- - -
-
- -
+
+
+
+
+
+ +
+
<% } %> - + From cc78e5acd5d4382371825eb80c9290493a70c3e8 Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Fri, 20 Dec 2019 01:02:57 +0100 Subject: [PATCH 3/5] Updated location to start from selected when edit. --- app/controllers/location.js | 15 ++++- app/views/location.ejs | 119 +++++++++++++++++++++--------------- 2 files changed, 83 insertions(+), 51 deletions(-) diff --git a/app/controllers/location.js b/app/controllers/location.js index 77ad699..214fcbf 100644 --- a/app/controllers/location.js +++ b/app/controllers/location.js @@ -4,10 +4,11 @@ const getLocation = async (req, res) => { const title = "Odaberite lokaciju"; const nextStep = req.query.nextStep || "/"; - //Check if location data already exists + //Check if location data already exists (active request) //If it does then get location is called through edit field query //and map should show already selected location not initial map - let selectedLatLngBounds = 0; + let selectedLatLngBounds = {}; + let boundsSelected = false; const searchRequest = await currentSearchRequest(req); @@ -18,13 +19,21 @@ const getLocation = async (req, res) => { const selectedArea = searchRequest.areaToSearch; const sw = selectedArea.coordinates[0][3]; const ne = selectedArea.coordinates[0][1]; + if (sw[0] && ne[0]) { - selectedLatLngBounds = [sw, ne]; + selectedLatLngBounds = { + swLat: sw[1], + swLng: sw[0], + neLat: ne[1], + neLng: ne[0] + }; + boundsSelected = true; } res.render("location", { nextStep, title, + boundsSelected, selectedLatLngBounds }); }; diff --git a/app/views/location.ejs b/app/views/location.ejs index dc587cb..8e30feb 100644 --- a/app/views/location.ejs +++ b/app/views/location.ejs @@ -50,15 +50,15 @@ function locateMe() { if (navigator.geolocation) { - - function onLocationSuccess (position) { - const coordinates = position && position.coords ? position.coords : null; - if (coordinates){ + function onLocationSuccess(position) { + const coordinates = + position && position.coords ? position.coords : null; + if (coordinates) { const longitude = coordinates.longitude || null; const latitude = coordinates.latitude || null; - if (longitude && latitude && map){ - map.setCenter({lat: latitude, lng: longitude}); + if (longitude && latitude && map) { + map.setCenter({ lat: latitude, lng: longitude }); map.setZoom(16); } } @@ -70,20 +70,20 @@ function initMap() { const BOSNIA_BOUNDS = { - north: 45.70, + north: 45.7, south: 41.69, west: 15.55, - east: 20.77, + east: 20.77 }; const SARAJEVO_COORDINATES = { lat: 43.85, - lng: 18.41, + lng: 18.41 }; - const mapElement = document.getElementById('map'); + const mapElement = document.getElementById("map"); const restrictMapPanningToBosniaOnly = { latLngBounds: BOSNIA_BOUNDS, - strictBounds: true, + strictBounds: true }; const initialMapParams = { center: SARAJEVO_COORDINATES, @@ -96,45 +96,50 @@ }; map = new google.maps.Map(mapElement, initialMapParams); - //After map initialization we check if area is already selected - //If yes we bound map to show selected area - console.log(<%= selectedLatLngBounds[0] %>); - if (<%= selectedLatLngBounds %>) { - map.fitBounds(<%= selectedLatLngBounds %>); - } - - const inputElement = document.getElementById('autocompleteInput'); - const restrictAutocompleteResultsToBosniaOnly = {'country': 'ba'}; + const inputElement = document.getElementById("autocompleteInput"); + const restrictAutocompleteResultsToBosniaOnly = { country: "ba" }; const initialAutocompleteParams = { - types: ['geocode'], + types: ["geocode"], componentRestrictions: restrictAutocompleteResultsToBosniaOnly, - fields: ['geometry', 'types', 'address_components'] + fields: ["geometry", "types", "address_components"] }; - autocomplete = new google.maps.places.Autocomplete(inputElement, initialAutocompleteParams); - autocomplete.bindTo('bounds', map); - autocomplete.addListener('place_changed', onPlaceChanged); + autocomplete = new google.maps.places.Autocomplete( + inputElement, + initialAutocompleteParams + ); + autocomplete.bindTo("bounds", map); + autocomplete.addListener("place_changed", onPlaceChanged); pacSelectFirst(inputElement); addLocateMeButton(map); + + //After map initialization we check if area is already selected + //If yes we bound map to show already selected area + const boundsSelected = <%- boundsSelected %>; + const selectedLatLngBounds = <%- JSON.stringify(selectedLatLngBounds) %>; + + if (boundsSelected) { + boundMapToSelected(map, selectedLatLngBounds); + } } function addLocateMeButton(map) { - var parent = document.createElement('div'); + var parent = document.createElement("div"); parent.className = "locate-me-container"; - var a = document.createElement('a'); + var a = document.createElement("a"); a.id = "locateMe"; a.className = "btn-floating"; - var i = document.createElement('i'); + var i = document.createElement("i"); i.innerText = "gps_fixed"; i.className = "material-icons right"; - a.appendChild(i) + a.appendChild(i); a.addEventListener("click", locateMe); - parent.appendChild(a) + parent.appendChild(a); - map.controls[google.maps.ControlPosition.RIGHT_BOTTOM].push(parent) + map.controls[google.maps.ControlPosition.RIGHT_BOTTOM].push(parent); } function onPlaceChanged() { @@ -149,33 +154,49 @@ function pacSelectFirst(input) { // store the original event binding function const _addEventListener = input.addEventListener - ? input.addEventListener - : input.attachEvent + ? input.addEventListener + : input.attachEvent; - function addEventListenerWrapper (type, listener) { + function addEventListenerWrapper(type, listener) { // Simulate a 'down arrow' keypress on hitting 'return' when no pac suggestion is selected, // and then trigger the original listener. - if (type == 'keydown') { - const originalListener = listener - listener = function (event) { - const suggestionSelected = $('.pac-item-selected').length > 0 - if (event.key == 'Enter' && !suggestionSelected) { - const simulatedDownArrow = $.Event('keydown', { + if (type == "keydown") { + const originalListener = listener; + listener = function(event) { + const suggestionSelected = $(".pac-item-selected").length > 0; + if (event.key == "Enter" && !suggestionSelected) { + const simulatedDownArrow = $.Event("keydown", { keyCode: 40, which: 40 - }) - originalListener.apply(input, [simulatedDownArrow]) + }); + originalListener.apply(input, [simulatedDownArrow]); } - originalListener.apply(input, [event]) - } + originalListener.apply(input, [event]); + }; } - _addEventListener.apply(input, [type, listener]) + _addEventListener.apply(input, [type, listener]); } - input.addEventListener = addEventListenerWrapper - input.attachEvent = addEventListenerWrapper + input.addEventListener = addEventListenerWrapper; + input.attachEvent = addEventListenerWrapper; + } + function boundMapToSelected(map, selectedLatLngBounds) { + + const swBound = new google.maps.LatLng( + selectedLatLngBounds.swLat, + selectedLatLngBounds.swLng + ); + const neBound = new google.maps.LatLng( + selectedLatLngBounds.neLat, + selectedLatLngBounds.neLng + ); + let bounds = new google.maps.LatLngBounds(); + bounds.extend(swBound); + bounds.extend(neBound); + + map.fitBounds(bounds); } $(document).ready(function() { @@ -187,7 +208,9 @@ $("#east").val(mapBounds.getNorthEast().lng()); $("#west").val(mapBounds.getSouthWest().lng()); - $("#locationInput").val(document.getElementById('autocompleteInput').value); + $("#locationInput").val( + document.getElementById("autocompleteInput").value + ); $("#form-map-output").submit(); }); From fed2dc00dc4d81d39b3c4c5fd06bb8ee6517c89f Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Sun, 29 Dec 2019 23:42:39 +0100 Subject: [PATCH 4/5] Changed number of rooms. --- .gitignore | 2 ++ app/crawler/specificCrawlers/rental.js | 13 ++++++------- test/rentalScrapeTest.js | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index b24fc61..02b0461 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ node_modules/ .env .idea/ +.eslintrc +.vscode/ diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js index 8f38dc8..f646b92 100644 --- a/app/crawler/specificCrawlers/rental.js +++ b/app/crawler/specificCrawlers/rental.js @@ -218,6 +218,9 @@ class RentalCrawler { const jsonData = scriptElement[0].children[0].data.substring(20); const parsedJsonData = JSON.parse(jsonData); extractedData = parsedJsonData[0]; + + ////**Trying to fix + console.log("Extracted data: ", extractedData); } catch (e) { throw { message: "Can't find ad data JSON" }; } @@ -312,7 +315,7 @@ class RentalCrawler { let numberOfRooms = parseInt(extractedData["re_realEstates_roomsNO"]) + - parseInt(extractedData["re_realEstates_bedroomNO"]) || null, + parseInt(extractedData["re_realEstates_bedNO"]) || null, numberOfFloors = parseInt(extractedData["re_realEstates_floorsNO"]) || this.getNumberOfFloorsFromFloorId(extractedData["re_floorNO_id"]), @@ -397,9 +400,7 @@ class RentalCrawler { ); if (!publishedDateMoment.isValid()) { throw { - message: `Invalid published date : ${ - extractedData["re_realEstates_inserted"] - }` + message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}` }; } @@ -410,9 +411,7 @@ class RentalCrawler { ); if (!renewedDateMoment.isValid()) { throw { - message: `Invalid renewed date : ${ - extractedData["re_realEstates_edited"] - }` + message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}` }; } diff --git a/test/rentalScrapeTest.js b/test/rentalScrapeTest.js index 9828f2d..4032c01 100644 --- a/test/rentalScrapeTest.js +++ b/test/rentalScrapeTest.js @@ -13,5 +13,5 @@ if (urlToScrape) { })(); } else { console.log("No URL to scrape. Use like this : "); - console.log("npm run test-olx-scraper -- URL_TO_SCRAPE"); + console.log("npm run test-rental-scraper -- URL_TO_SCRAPE"); } From 0c2d218d29f31c46bf10d06d8caecce83eacb18a Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Thu, 2 Jan 2020 00:10:31 +0100 Subject: [PATCH 5/5] Changed floor numbers and basement-attic tag. --- app/crawler/specificCrawlers/rental.js | 43 +++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js index f646b92..39eb1c5 100644 --- a/app/crawler/specificCrawlers/rental.js +++ b/app/crawler/specificCrawlers/rental.js @@ -218,9 +218,6 @@ class RentalCrawler { const jsonData = scriptElement[0].children[0].data.substring(20); const parsedJsonData = JSON.parse(jsonData); extractedData = parsedJsonData[0]; - - ////**Trying to fix - console.log("Extracted data: ", extractedData); } catch (e) { throw { message: "Can't find ad data JSON" }; } @@ -355,7 +352,9 @@ class RentalCrawler { realEstatePropertiesFromInfrastructure.phoneConnection, cableTV = realEstatePropertiesFromInfrastructure.cableTV, internet = realEstatePropertiesFromInfrastructure.internet, - basementAttic = realEstatePropertiesFromSpaces.basementAttic, + basementAttic = + realEstatePropertiesFromSpaces.basementAttic || + this.checkBasemAtticFromFloors(extractedData["re_floorNO_id"]), storeRoom = realEstatePropertiesFromSpaces.storeRoom, videoSurveillance = realEstatePropertiesFromDescriptions.videoSurveillance || @@ -781,8 +780,42 @@ class RentalCrawler { if (floorIds.length === 0) { return null; } + let noOfFloors = floorIds.length; + // Floors of 'suteren', 'podrum', 'tavan' and 'potkrovlje' are not counted + floorIds.forEach(id => { + if ( + parseInt(id) === 1 || + parseInt(id) === 2 || + parseInt(id) === 12 || + parseInt(id) === 14 + ) { + noOfFloors--; + } + }); + return noOfFloors; + } - return floorIds.length; + checkBasemAtticFromFloors(floorsIdText) { + // floorIdText can be array of numbers, separated by comma or number + const floorIds = floorsIdText.split(","); + + let check = false; + + if (floorIds.length === 0) { + check = false; + } + //If floors 'suteren', 'podrum', 'tavan' and 'potkrovlje' exists then tag for basement-attic is true + floorIds.forEach(id => { + if ( + parseInt(id) === 1 || + parseInt(id) === 2 || + parseInt(id) === 12 || + parseInt(id) === 14 + ) { + check = true; + } + }); + return check; } async sleep(ms) {