senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

refactor crawler - adapt to use new ENUM objects

Browse Source
This commit is contained in:
Bilal Catic
2019-09-30 10:27:12 +02:00
parent e1dfd6a4eb
commit 9c0104a57c
2 changed files with 13 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/crawler/crawl.js
View File

@@ -5,8 +5,6 @@
All environment specific configuration is read here and All environment specific configuration is read here and
passed to the crawlers and savers. passed to the crawlers and savers.
*/ */
require("dotenv").config();
const OlxCrawler = require("./specific/olx"); const OlxCrawler = require("./specific/olx");
const { OLX_CONFIG } = require("./crawlerConfig"); const { OLX_CONFIG } = require("./crawlerConfig");
const PostgresSaver = require("./savers/postgres"); const PostgresSaver = require("./savers/postgres");

29
app/crawler/specific/olx.js
View File

@@ -22,12 +22,12 @@ const OLX_ENUMS = {
[CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje" [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje"
}, },
OLX_AD_CATEGORY: { OLX_AD_CATEGORY: {
[AD_CATEGORY.CATEGORY_FLAT]: "&kategorija=23", [AD_CATEGORY.FLAT]: "&kategorija=23",
[AD_CATEGORY.CATEGORY_HOUSE]: "&kategorija=24", [AD_CATEGORY.HOUSE]: "&kategorija=24",
[AD_CATEGORY.CATEGORY_LAND]: "&kategorija=29", //[AD_CATEGORY.LAND]: "&kategorija=29",
[AD_CATEGORY.CATEGORY_OFFICE]: "&kategorija=25", //[AD_CATEGORY.OFFICE]: "&kategorija=25",
[AD_CATEGORY.CATEGORY_APARTMENT]: "&kategorija=27", [AD_CATEGORY.APARTMENT]: "&kategorija=27"
[AD_CATEGORY.CATEGORY_GARAGE]: "&kategorija=30" //[AD_CATEGORY.CATEGORY_GARAGE]: "&kategorija=30"
}, },
MAX_DETAIL_FIELDS: 30, MAX_DETAIL_FIELDS: 30,
OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm", OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
@@ -38,10 +38,7 @@ class OlxCrawler {
constructor( constructor(
savers = [], savers = [],
crawlerAdTypes = CRAWLER_AD_TYPE.ALL, crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
crawlerAdCategories = [ crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
AD_CATEGORY.CATEGORY_FLAT,
AD_CATEGORY.CATEGORY_HOUSE
],
maxPages = 1000, maxPages = 1000,
maxResultsPerPage = 100, maxResultsPerPage = 100,
ignoredUsernames = [], ignoredUsernames = [],
@@ -407,7 +404,7 @@ class OlxCrawler {
url, url,
agencyObjectId: olxId, agencyObjectId: olxId,
originAgencyName: AD_AGENCY.OLX, originAgencyName: AD_AGENCY.OLX,
realEstateType: this.getAdCategoryId(category), realEstateType: parsedCategory,
adType: parsedAdType, adType: parsedAdType,
title, title,
price: parsedPrice, price: parsedPrice,
@@ -448,15 +445,15 @@ class OlxCrawler {
getAdCategoryId(categoryText) { getAdCategoryId(categoryText) {
switch (categoryText) { switch (categoryText) {
case "Stanovi": case "Stanovi":
return AD_CATEGORY.CATEGORY_FLAT; return AD_CATEGORY.FLAT.id;
case "Zemljišta": case "Zemljišta":
return AD_CATEGORY.CATEGORY_LAND; return undefined; //AD_CATEGORY.LAND;
case "Kuće": case "Kuće":
return AD_CATEGORY.CATEGORY_HOUSE; return AD_CATEGORY.HOUSE.id;
case "Poslovni prostori": case "Poslovni prostori":
return AD_CATEGORY.CATEGORY_OFFICE; return undefined; //AD_CATEGORY.OFFICE;
case "Apartmani": case "Apartmani":
return AD_CATEGORY.CATEGORY_APARTMENT; return AD_CATEGORY.APARTMENT.id;
default: default:
return undefined; return undefined;
} }