diff --git a/backend/build/server.js b/backend/build/server.js index 9645ef4..44f22d0 100644 --- a/backend/build/server.js +++ b/backend/build/server.js @@ -74,7 +74,7 @@ router.get('/search', function () { var _ref = _asyncToGenerator(regeneratorRuntime.mark(function _callee(req, res, next) { - var bounds, minPrice, maxPrice, minSize, maxSize, rooms, adType, properties, query, _bounds$split$map, _bounds$split$map2, lat1, lng1, lat2, lng2, box, price, allRooms, or, size, all; + var bounds, minPrice, maxPrice, minSize, maxSize, rooms, adType, category, sort, properties, query, _bounds$split$map, _bounds$split$map2, lat1, lng1, lat2, lng2, box, price, allRooms, or, size, allCategories, _or, all; return regeneratorRuntime.wrap(function _callee$(_context) { while (1) { @@ -88,6 +88,8 @@ maxSize = req.query.maxSize; rooms = req.query.rooms; adType = req.query.adType; + category = req.query.category; + sort = req.query.sort; properties = db.collection('listings'); query = {}; @@ -165,35 +167,49 @@ }); } + if (category) { + allCategories = category.split(','); + _or = allCategories.map(function (val) { + return { + category: parseInt(val) + }; + }); + + + query = Object.assign(query, { + "$or": _or + }); + } + console.log('QUERY: ', query); - _context.next = 18; + _context.next = 21; return properties.find(query, { //"sort": [['field1','asc'], ['field2','desc']] "sort": [['price', 'asc']] }).toArray(); - case 18: + case 21: all = _context.sent; res.json(all); res.end(); - _context.next = 27; + _context.next = 30; break; - case 23: - _context.prev = 23; + case 26: + _context.prev = 26; _context.t0 = _context['catch'](0); console.log('error:', _context.t0); next(_context.t0); - case 27: + case 30: case 'end': return _context.stop(); } } - }, _callee, undefined, [[0, 23]]); + }, _callee, undefined, [[0, 26]]); })); return function (_x, _x2, _x3) { diff --git a/backend/server.js b/backend/server.js index 2646427..a393ded 100644 --- a/backend/server.js +++ b/backend/server.js @@ -21,6 +21,8 @@ router.get('/search', async (req, res, next) => { const maxSize = req.query.maxSize; const rooms = req.query.rooms; const adType = req.query.adType; + const category = req.query.category; + const sort = req.query.sort; const properties = db.collection('listings'); let query = {}; @@ -93,6 +95,19 @@ router.get('/search', async (req, res, next) => { }); } + if (category) { + const allCategories = category.split(','); + const or = allCategories.map(val => { + return { + category: parseInt(val) + }; + }); + + query = Object.assign(query, { + "$or": or + }); + } + console.log('QUERY: ', query); const all = await properties.find(query, { //"sort": [['field1','asc'], ['field2','desc']] diff --git a/crawler/enums.js b/crawler/enums.js index e2b6c13..bc79e63 100644 --- a/crawler/enums.js +++ b/crawler/enums.js @@ -3,3 +3,8 @@ export const AD_TYPE_RENT = 2; export const IGNORED_USERNAMES = ['rental'] +export const CATEGORY_FLAT = 0; +export const CATEGORY_HOUSE = 1; +export const CATEGORY_OFFICE = 2; +export const CATEGORY_LAND = 3; + diff --git a/crawler/savers/mongo.js b/crawler/savers/mongo.js index f795688..2bff4ee 100644 --- a/crawler/savers/mongo.js +++ b/crawler/savers/mongo.js @@ -32,7 +32,9 @@ export default class MongoSaver { let resultsForMongo = Object.keys(results).map((key) => { return results[key] }); - this.collection.insert(resultsForMongo); + + this.collection.update({ url: results.url }, resultsForMongo, { upsert: true }); + // this.collection.insert(resultsForMongo); } async close() { diff --git a/crawler/specific/olx.js b/crawler/specific/olx.js index e8aa542..02a195e 100644 --- a/crawler/specific/olx.js +++ b/crawler/specific/olx.js @@ -3,7 +3,15 @@ let fetch = require('node-fetch'); let cheerio = require('cheerio'); let fs = require('fs'); -import {AD_TYPE_SALE, IGNORED_USERNAMES} from '../enums'; + +import { + AD_TYPE_SALE, + IGNORED_USERNAMES, + CATEGORY_FLAT, + CATEGORY_HOUSE, + CATEGORY_OFFICE, + CATEGORY_LAND +} from '../enums'; export default class OlxCrawler { @@ -26,6 +34,8 @@ export default class OlxCrawler { } const title = $('#naslovartikla').text(); + const category = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text(); + const price = $('#pc > p:nth-child(2)').text(); const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text(); const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text(); @@ -64,13 +74,21 @@ export default class OlxCrawler { } const parsedPrice = parsePrice(price); + let parsedRooms; + + if (rooms === 'Garsonjera') { + parsedRooms = 0; + } else { + parsedRooms = parseRooms(rooms); + } const data = { + category: this.getCategoryId(category), url, title, - price: isNaN(parsedPrice) || price, + price: isNaN(parsedPrice) ? price : parsedPrice, size: parseFloat(size), - rooms: parseRooms(rooms), + rooms: parsedRooms, floor: parseInt(floor), address, location, @@ -127,6 +145,18 @@ export default class OlxCrawler { } } + getCategoryId (category) { + if (category === 'Stanovi') { + return CATEGORY_FLAT; + } else if (category === 'Zemljišta') { + return CATEGORY_LAND; + } else if (category === 'Kuće') { + return CATEGORY_HOUSE; + } else if (category === 'Poslovni prostori') { + return CATEGORY_OFFICE; + } + } + async sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } diff --git a/web/components/Filters.js b/web/components/Filters.js index 63fad2f..b275f3b 100644 --- a/web/components/Filters.js +++ b/web/components/Filters.js @@ -1,5 +1,11 @@ import React from "react"; import { formatFilterNumber } from "../lib/helpers"; +import { + CATEGORY_FLAT, + CATEGORY_HOUSE, + CATEGORY_OFFICE, + CATEGORY_LAND +} from '../../crawler/enums'; export default class Filters extends React.Component { onCloseClick(e) { @@ -40,6 +46,10 @@ export default class Filters extends React.Component { this.props.dispatch({type: 'SET_ROOMS', action: {rooms}}); } + onCategoryClick(category) { + this.props.dispatch({type: 'SET_CATEGORY', action: {category}}); + } + onRefreshClick() { this.updateSearch(); } @@ -57,6 +67,7 @@ export default class Filters extends React.Component { render() { const { filters } = this.props; const selectedRooms = val => filters.rooms[val] ? "selected" : ""; + const selectedCategory = val => filters.category[val] ? "selected": ""; return (
@@ -105,18 +116,26 @@ export default class Filters extends React.Component { TIP
-
+
Stan
-
+
Kuća
-
+
Zemljište
-
+
Poslovni prostor
@@ -155,9 +174,9 @@ export default class Filters extends React.Component {
Garsonjera diff --git a/web/components/Main.js b/web/components/Main.js index 09b0c5b..7551d16 100644 --- a/web/components/Main.js +++ b/web/components/Main.js @@ -14,7 +14,8 @@ class Main extends React.Component { listings: (new Map()), imageIndex: 0, filters: { - rooms: {} + rooms: {}, + category: {} } }; } @@ -99,7 +100,14 @@ class Main extends React.Component { refreshListings() { const map = this.map; - const {rooms, minSize, maxSize, minPrice, maxPrice} = this.state.filters; + const { + rooms, + minSize, + maxSize, + minPrice, + maxPrice, + category + } = this.state.filters; const properties = loadProperties({ bounds: map.getBounds().toUrlValue(), @@ -107,7 +115,8 @@ class Main extends React.Component { minSize, maxSize, minPrice, - maxPrice + maxPrice, + category }); properties.then(p=> p.text()).then(p => { diff --git a/web/lib/api.js b/web/lib/api.js index 9e9851f..439d1b6 100644 --- a/web/lib/api.js +++ b/web/lib/api.js @@ -4,18 +4,24 @@ export const loadProperties = ({ bounds, minPrice = '', maxPrice = '', - minSize = '', - maxSize = '', - rooms + minSize = '', + maxSize = '', + rooms = {}, + category = {} }) => { const allRooms = Object .keys(rooms) .filter((v) => rooms[v]) .join(','); + const allCategories = Object + .keys(category) + .filter((v) => category[v]) + .join(','); + // TODO: handle errors //return fetch(process.env.API_URL + '/api/search', { - return fetch(`http://localhost:3001/api/search?bounds=${bounds}&minPrice=${minPrice}&maxPrice=${maxPrice}&rooms=${allRooms}&minSize=${minSize}&maxSize=${maxSize}`, { + return fetch(`http://localhost:3001/api/search?bounds=${bounds}&minPrice=${minPrice}&maxPrice=${maxPrice}&rooms=${allRooms}&minSize=${minSize}&maxSize=${maxSize}&category=${allCategories}`, { //credentials: 'include' }); diff --git a/web/lib/handlers.js b/web/lib/handlers.js index 14c1d5f..de3b25a 100644 --- a/web/lib/handlers.js +++ b/web/lib/handlers.js @@ -99,7 +99,6 @@ const searchPlaceChanged = ({ type, action }, component) => { const setRooms = ({ type, action }, component) => { const prevRooms = component.state.filters.rooms || {}; - console.log('BERORE ROOMS'); component.setState( { @@ -112,14 +111,12 @@ const setRooms = ({ type, action }, component) => { } }, () => { - console.log('after rooms'); component.refreshListings(); } ); }; const updateSearch = ({ type, action }, component) => { - console.log("updating search"); component.setState( { filters: { @@ -134,6 +131,26 @@ const updateSearch = ({ type, action }, component) => { ); }; +const setCategory = ({type, action}, component) => { + + const prevCategory = component.state.filters.category || {}; + + component.setState( + { + filters: { + ...component.state.filters, + category: { + ...prevCategory, + [action.category]: !prevCategory[action.category] + } + } + }, + () => { + component.refreshListings(); + } + ); +} + const handlers = { SET_MIN_PRICE: setMinPrice, SET_MAX_PRICE: setMaxPrice, @@ -147,7 +164,8 @@ const handlers = { SEARCH_PLACE_CHANGED: searchPlaceChanged, SET_ROOMS: setRooms, VIEW_LISTING_DETAILS: viewListingDetails, - UPDATE_SEARCH: updateSearch + UPDATE_SEARCH: updateSearch, + SET_CATEGORY: setCategory }; export const handleMessage = ({ type, action }, component) => { diff --git a/web/lib/helpers.js b/web/lib/helpers.js index 25ddffb..7fe59a1 100644 --- a/web/lib/helpers.js +++ b/web/lib/helpers.js @@ -1,5 +1,5 @@ export const formatPrice = (p) => { - if (p === -1) { + if (isNaN(p)) { return 'Po dogovoru' }