senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

add ALL category option for Rental agency

Browse Source
This commit is contained in:
Bilal Catic
2019-10-28 09:24:08 +01:00
parent 2c2fcd648f
commit 1e36cb8423
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/crawler/specificCrawlers/rental.js
View File

@@ -26,6 +26,7 @@ const RENTAL_ENUMS = {
[CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2" [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
}, },
RENTAL_AD_CATEGORY: { RENTAL_AD_CATEGORY: {
[AD_CATEGORY.ALL.id]: "",
[AD_CATEGORY.FLAT.id]: "/tip-2", [AD_CATEGORY.FLAT.id]: "/tip-2",
[AD_CATEGORY.HOUSE.id]: "/tip-1", [AD_CATEGORY.HOUSE.id]: "/tip-1",
[AD_CATEGORY.LAND.id]: "/tip-5", [AD_CATEGORY.LAND.id]: "/tip-5",
@@ -137,7 +138,7 @@ class RentalCrawler {
const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes]; const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory]; const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
if (urlAdTypePart && urlCategoryPart) { if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
while (true) { while (true) {
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`; const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
const singlePageResults = await this.indexSinglePage( const singlePageResults = await this.indexSinglePage(
@@ -204,7 +205,7 @@ class RentalCrawler {
} }
async scrapeAd(url) { async scrapeAd(url) {
// console.log("[RENTAL] Scraping : ", url); console.log("[RENTAL] Scraping : ", url);
try { try {
const adPageSource = await fetch(url); const adPageSource = await fetch(url);
const body = await adPageSource.text(); const body = await adPageSource.text();