From 5e8e13a984852db2476f06892a1ea8174a5a3e65 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 30 Sep 2019 14:27:01 +0200 Subject: [PATCH] fix enums --- app/crawler/crawlerConfig.js | 6 ++++-- app/crawler/specific/olx.js | 14 +++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/app/crawler/crawlerConfig.js b/app/crawler/crawlerConfig.js index eac99c6..3b2abef 100644 --- a/app/crawler/crawlerConfig.js +++ b/app/crawler/crawlerConfig.js @@ -12,7 +12,7 @@ const olxParsedCrawlerAdCategories = ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category => category.trim() ) - : ["CATEGORY_FLAT", "CATEGORY_HOUSE"]; + : ["FLAT", "HOUSE"]; const olxIgnoredUsernames = process.env.OLX_IGNORED_USERNAMES !== undefined @@ -22,7 +22,9 @@ const olxIgnoredUsernames = : []; const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories - .map(categoryName => AD_CATEGORY[categoryName]) + .map(categoryName => + AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined + ) .filter(category => !!category); const OLX_CONFIG = { diff --git a/app/crawler/specific/olx.js b/app/crawler/specific/olx.js index 0b2ee49..cbeacbc 100644 --- a/app/crawler/specific/olx.js +++ b/app/crawler/specific/olx.js @@ -22,12 +22,12 @@ const OLX_ENUMS = { [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje" }, OLX_AD_CATEGORY: { - [AD_CATEGORY.FLAT]: "&kategorija=23", - [AD_CATEGORY.HOUSE]: "&kategorija=24", - //[AD_CATEGORY.LAND]: "&kategorija=29", - //[AD_CATEGORY.OFFICE]: "&kategorija=25", - [AD_CATEGORY.APARTMENT]: "&kategorija=27" - //[AD_CATEGORY.CATEGORY_GARAGE]: "&kategorija=30" + [AD_CATEGORY.FLAT.id]: "&kategorija=23", + [AD_CATEGORY.HOUSE.id]: "&kategorija=24", + //[AD_CATEGORY.LAND.id]: "&kategorija=29", + //[AD_CATEGORY.OFFICE.id]: "&kategorija=25", + [AD_CATEGORY.APARTMENT.id]: "&kategorija=27" + //[AD_CATEGORY.CATEGORY_GARAGE.id]: "&kategorija=30" }, MAX_DETAIL_FIELDS: 30, OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm", @@ -193,7 +193,7 @@ class OlxCrawler { } async scrapeAd(url) { - //console.log("Scraping : ", url); + // console.log("Scraping : ", url); try { const adPageSource = await fetch(url); const body = await adPageSource.text();