diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 90e1543..dc4f990 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -27,8 +27,8 @@ module.exports = class OlxCrawler { async crawl() { const filteredResults = []; - const realestateRequests = await allRERequest(); - const urls = this.createRequestUrls(realestateRequests); + const realEstateRequests = await allRERequest(); + const urls = this.createRequestUrls(realEstateRequests); let results = await this.indexPages( urls, this.fromPage, @@ -65,24 +65,94 @@ module.exports = class OlxCrawler { return []; } - createRequestUrls(realestateRequests) { + createRequestUrls(realEstateRequests) { const urls = []; - for (const request of realestateRequests) { - const realsestateType = - "kategorija=" + - getRealEstateTypeEnum(request.realEstateType).olxCategory; - const region = "kanton=" + getRegion(request.region).olxid; - const municipality = - "grad%5B%5D=" + - getMunicipality(request.region, request.municipality).olxid; - const sizeMin = "kvadrata_min=" + request.sizeMin; - const sizeMax = "kvadrata_max=" + request.sizeMax; - const priceMin = "od=" + request.priceMin; - const priceMax = "do=" + request.priceMax; + for (const request of realEstateRequests) { + const { + realEstateType, + region, + sizeMin, + sizeMax, + priceMin, + priceMax + } = request; + + const urlRealEstateParams = [ + { + paramName: "kanton", + paramValue: region, + useParam: false + }, + { + paramName: "kategorija", + paramValue: getRealEstateTypeEnum(realEstateType).olxid, + useParam: true + }, + { + paramName: "kvadrata_min", + paramValue: sizeMin, + useParam: true + }, + { + paramName: "kvadrata_max", + paramValue: sizeMax, + useParam: true + }, + { + paramName: "od", + paramValue: priceMin, + useParam: true + }, + { + paramName: "do", + paramValue: priceMax, + useParam: true + } + ]; + const urlResultsParams = [ + { + paramName: "vrstapregleda", + paramValue: "tabela", + useParam: true + }, + { + paramName: "sort_order", + paramValue: "desc", + useParam: true + }, + { + paramName: "vrsta", + paramValue: "samoprodaja", + useParam: true + }, + { + paramName: "stranica", + paramValue: "0", + useParam: true + } + ]; + + const paramsReduceFunction = (accumulatedValue, currentParam) => { + const { paramName, paramValue, useParam } = currentParam; + if (useParam) { + return `${accumulatedValue}&${paramName}=${paramValue}`; + } else { + return accumulatedValue; + } + }; + + const reducedRealEstateParams = urlRealEstateParams.reduce( + paramsReduceFunction, + "" + ); + const reducedResultsParams = urlResultsParams.reduce( + paramsReduceFunction, + "" + ); const olxUrl = { - url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`, + url: `https://www.olx.ba/pretraga?${reducedRealEstateParams}${reducedResultsParams}`, email: request.email, uuid: request.uniqueId, hrefs: this.hrefs diff --git a/index.js b/index.js index b9ea96c..b08652d 100644 --- a/index.js +++ b/index.js @@ -6,11 +6,6 @@ const { getRealEstateTypes, postRealEstateTypes } = require("./app/controllers/realEstateTypes"); -const { getRegion, postRegion } = require("./app/controllers/regions"); -const { - getMunicipality, - postMunicipality -} = require("./app/controllers/municipalities"); const { getSize, postSize } = require("./app/controllers/sizes"); const { getGardenSize, @@ -26,10 +21,7 @@ const { postQuerySubmit } = require("./app/controllers/querySubmit"); const { getGoAgain } = require("./app/controllers/goAgain"); -const { - getNeighborhood, - postNeighborhood -} = require("./app/controllers/neighborhoodMap"); +const { getLocation, postLocation } = require("./app/controllers/location"); const { getUnsubscribe } = require("./app/controllers/unsubscribe"); const { getRealEstates } = require("./app/controllers/realEstates"); const { redirect } = require("./app/controllers/redirect"); @@ -147,14 +139,8 @@ app.get("/vrstanekretnine", getRealEstateTypes); app.post("/vrstanekretnine/:request_id", postRealEstateTypes); app.post("/vrstanekretnine", postRealEstateTypes); -app.get("/grad/:request_id", getRegion); -app.post("/grad/:request_id", postRegion); - -app.get("/mjesto/:request_id", getMunicipality); -app.post("/mjesto/:request_id", postMunicipality); - -app.get("/naselje/:request_id/:municipality", getNeighborhood); -app.post("/naselje/:request_id/:municipality", postNeighborhood); +app.get("/lokacija/:request_id", getLocation); +app.post("/lokacija/:request_id", postLocation); app.get("/povrsina/:request_id", getSize); app.post("/povrsina/:request_id", postSize);