senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

fix enums

Browse Source
This commit is contained in:
Bilal Catic
2019-09-30 14:27:01 +02:00
parent 37ba7e2c8c
commit 5e8e13a984
2 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/crawler/crawlerConfig.js
View File

@@ -12,7 +12,7 @@ const olxParsedCrawlerAdCategories =
? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category => ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim() category.trim()
) )
: ["CATEGORY_FLAT", "CATEGORY_HOUSE"]; : ["FLAT", "HOUSE"];
const olxIgnoredUsernames = const olxIgnoredUsernames =
process.env.OLX_IGNORED_USERNAMES !== undefined process.env.OLX_IGNORED_USERNAMES !== undefined
@@ -22,7 +22,9 @@ const olxIgnoredUsernames =
: []; : [];
const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories
.map(categoryName => AD_CATEGORY[categoryName]) .map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category); .filter(category => !!category);
const OLX_CONFIG = { const OLX_CONFIG = {

14
app/crawler/specific/olx.js
View File

@@ -22,12 +22,12 @@ const OLX_ENUMS = {
[CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje" [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje"
}, },
OLX_AD_CATEGORY: { OLX_AD_CATEGORY: {
[AD_CATEGORY.FLAT]: "&kategorija=23", [AD_CATEGORY.FLAT.id]: "&kategorija=23",
[AD_CATEGORY.HOUSE]: "&kategorija=24", [AD_CATEGORY.HOUSE.id]: "&kategorija=24",
//[AD_CATEGORY.LAND]: "&kategorija=29", //[AD_CATEGORY.LAND.id]: "&kategorija=29",
//[AD_CATEGORY.OFFICE]: "&kategorija=25", //[AD_CATEGORY.OFFICE.id]: "&kategorija=25",
[AD_CATEGORY.APARTMENT]: "&kategorija=27" [AD_CATEGORY.APARTMENT.id]: "&kategorija=27"
//[AD_CATEGORY.CATEGORY_GARAGE]: "&kategorija=30" //[AD_CATEGORY.CATEGORY_GARAGE.id]: "&kategorija=30"
}, },
MAX_DETAIL_FIELDS: 30, MAX_DETAIL_FIELDS: 30,
OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm", OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
@@ -193,7 +193,7 @@ class OlxCrawler {
} }
async scrapeAd(url) { async scrapeAd(url) {
//console.log("Scraping : ", url); // console.log("Scraping : ", url);
try { try {
const adPageSource = await fetch(url); const adPageSource = await fetch(url);
const body = await adPageSource.text(); const body = await adPageSource.text();