From 1e36cb8423a267b7ee7b12f0875862c1ffb450e7 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 28 Oct 2019 09:24:08 +0100 Subject: [PATCH] add ALL category option for Rental agency --- app/crawler/specificCrawlers/rental.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js index 45e572d..b73278e 100644 --- a/app/crawler/specificCrawlers/rental.js +++ b/app/crawler/specificCrawlers/rental.js @@ -26,6 +26,7 @@ const RENTAL_ENUMS = { [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2" }, RENTAL_AD_CATEGORY: { + [AD_CATEGORY.ALL.id]: "", [AD_CATEGORY.FLAT.id]: "/tip-2", [AD_CATEGORY.HOUSE.id]: "/tip-1", [AD_CATEGORY.LAND.id]: "/tip-5", @@ -137,7 +138,7 @@ class RentalCrawler { const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes]; const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory]; - if (urlAdTypePart && urlCategoryPart) { + if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) { while (true) { const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`; const singlePageResults = await this.indexSinglePage( @@ -204,7 +205,7 @@ class RentalCrawler { } async scrapeAd(url) { - // console.log("[RENTAL] Scraping : ", url); + console.log("[RENTAL] Scraping : ", url); try { const adPageSource = await fetch(url); const body = await adPageSource.text();