remove region and municipality params when searching olx

app/helpers/crawlers/olxClawler.js

@@ -27,8 +27,8 @@ module.exports = class OlxCrawler {
 
   async crawl() {
     const filteredResults = [];
-    const realestateRequests = await allRERequest();
+    const realEstateRequests = await allRERequest();
-    const urls = this.createRequestUrls(realestateRequests);
+    const urls = this.createRequestUrls(realEstateRequests);
     let results = await this.indexPages(
       urls,
       this.fromPage,
@@ -65,24 +65,94 @@ module.exports = class OlxCrawler {
     return [];
   }
 
-  createRequestUrls(realestateRequests) {
+  createRequestUrls(realEstateRequests) {
     const urls = [];
 
-    for (const request of realestateRequests) {
+    for (const request of realEstateRequests) {
-      const realsestateType =
+      const {
-        "kategorija=" +
+        realEstateType,
-        getRealEstateTypeEnum(request.realEstateType).olxCategory;
+        region,
-      const region = "kanton=" + getRegion(request.region).olxid;
+        sizeMin,
-      const municipality =
+        sizeMax,
-        "grad%5B%5D=" +
+        priceMin,
-        getMunicipality(request.region, request.municipality).olxid;
+        priceMax
-      const sizeMin = "kvadrata_min=" + request.sizeMin;
+      } = request;
-      const sizeMax = "kvadrata_max=" + request.sizeMax;
+
-      const priceMin = "od=" + request.priceMin;
+      const urlRealEstateParams = [
-      const priceMax = "do=" + request.priceMax;
+        {
+          paramName: "kanton",
+          paramValue: region,
+          useParam: false
+        },
+        {
+          paramName: "kategorija",
+          paramValue: getRealEstateTypeEnum(realEstateType).olxid,
+          useParam: true
+        },
+        {
+          paramName: "kvadrata_min",
+          paramValue: sizeMin,
+          useParam: true
+        },
+        {
+          paramName: "kvadrata_max",
+          paramValue: sizeMax,
+          useParam: true
+        },
+        {
+          paramName: "od",
+          paramValue: priceMin,
+          useParam: true
+        },
+        {
+          paramName: "do",
+          paramValue: priceMax,
+          useParam: true
+        }
+      ];
+      const urlResultsParams = [
+        {
+          paramName: "vrstapregleda",
+          paramValue: "tabela",
+          useParam: true
+        },
+        {
+          paramName: "sort_order",
+          paramValue: "desc",
+          useParam: true
+        },
+        {
+          paramName: "vrsta",
+          paramValue: "samoprodaja",
+          useParam: true
+        },
+        {
+          paramName: "stranica",
+          paramValue: "0",
+          useParam: true
+        }
+      ];
+
+      const paramsReduceFunction = (accumulatedValue, currentParam) => {
+        const { paramName, paramValue, useParam } = currentParam;
+        if (useParam) {
+          return `${accumulatedValue}&${paramName}=${paramValue}`;
+        } else {
+          return accumulatedValue;
+        }
+      };
+
+      const reducedRealEstateParams = urlRealEstateParams.reduce(
+        paramsReduceFunction,
+        ""
+      );
+      const reducedResultsParams = urlResultsParams.reduce(
+        paramsReduceFunction,
+        ""
+      );
 
       const olxUrl = {
-        url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`,
+        url: `https://www.olx.ba/pretraga?${reducedRealEstateParams}${reducedResultsParams}`,
         email: request.email,
         uuid: request.uniqueId,
         hrefs: this.hrefs

index.js

@@ -6,11 +6,6 @@ const {
   getRealEstateTypes,
   postRealEstateTypes
 } = require("./app/controllers/realEstateTypes");
-const { getRegion, postRegion } = require("./app/controllers/regions");
-const {
-  getMunicipality,
-  postMunicipality
-} = require("./app/controllers/municipalities");
 const { getSize, postSize } = require("./app/controllers/sizes");
 const {
   getGardenSize,
@@ -26,10 +21,7 @@ const {
   postQuerySubmit
 } = require("./app/controllers/querySubmit");
 const { getGoAgain } = require("./app/controllers/goAgain");
-const {
+const { getLocation, postLocation } = require("./app/controllers/location");
-  getNeighborhood,
-  postNeighborhood
-} = require("./app/controllers/neighborhoodMap");
 const { getUnsubscribe } = require("./app/controllers/unsubscribe");
 const { getRealEstates } = require("./app/controllers/realEstates");
 const { redirect } = require("./app/controllers/redirect");
@@ -147,14 +139,8 @@ app.get("/vrstanekretnine", getRealEstateTypes);
 app.post("/vrstanekretnine/:request_id", postRealEstateTypes);
 app.post("/vrstanekretnine", postRealEstateTypes);
 
-app.get("/grad/:request_id", getRegion);
+app.get("/lokacija/:request_id", getLocation);
-app.post("/grad/:request_id", postRegion);
+app.post("/lokacija/:request_id", postLocation);
-
-app.get("/mjesto/:request_id", getMunicipality);
-app.post("/mjesto/:request_id", postMunicipality);
-
-app.get("/naselje/:request_id/:municipality", getNeighborhood);
-app.post("/naselje/:request_id/:municipality", postNeighborhood);
 
 app.get("/povrsina/:request_id", getSize);
 app.post("/povrsina/:request_id", postSize);