senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

move specific crawler config to the separated files

Browse Source
This commit is contained in:
Bilal Catic
2019-10-24 16:57:23 +02:00
parent e82a0cfba4
commit 935ae60ae1
7 changed files with 75 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/crawler/crawl.js
View File

@@ -5,8 +5,8 @@
All environment specific configuration is read here and
passed to the crawlers and savers.
*/
const OlxCrawler = require("./specific/olx");
const RentalCrawler = require("./specific/rental");
const OlxCrawler = require("./specificCrawlers/olx");
const RentalCrawler = require("./specificCrawlers/rental");
const { OLX_CONFIG, RENTAL_CONFIG } = require("./crawlerConfig");
const PostgresSaver = require("./savers/postgres");

68
app/crawler/crawlerConfig.js
View File

@@ -1,72 +1,8 @@
"use strict";
require("dotenv").config({ path: __dirname + "/./../../.env" });
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../common/enums");
const olxCrawlerAdType =
process.env.OLX_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
: null;
const rentalCrawlerAdType =
process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
: null;
const olxParsedCrawlerAdCategories =
process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const rentalParsedCrawlerAdCategories =
process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const olxIgnoredUsernames =
process.env.OLX_IGNORED_USERNAMES !== undefined
? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
username.trim()
)
: [];
const rentalIgnoredUsernames = [];
const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
const OLX_CONFIG = {
OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
OLX_MAX_RESULTS_PER_PAGE:
parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
};
const RENTAL_CONFIG = {
RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
RENTAL_MAX_RESULTS_PER_PAGE:
parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
RENTAL_DELAY_BETWEEN_PAGES:
parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000
};
const OLX_CONFIG = require("./specificConfigs/olx");
const RENTAL_CONFIG = require("./specificConfigs/rental");
module.exports = {
OLX_CONFIG,

37
app/crawler/specificConfigs/olx.js Normal file
View File

@@ -0,0 +1,37 @@
"use strict";
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
const olxCrawlerAdType =
process.env.OLX_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
: null;
const olxParsedCrawlerAdCategories =
process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const olxIgnoredUsernames =
process.env.OLX_IGNORED_USERNAMES !== undefined
? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
username.trim()
)
: [];
const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
module.exports = {
OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
OLX_MAX_RESULTS_PER_PAGE:
parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
};

33
app/crawler/specificConfigs/rental.js Normal file
View File

@@ -0,0 +1,33 @@
"use strict";
const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
const rentalCrawlerAdType =
process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
: null;
const rentalParsedCrawlerAdCategories =
process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
category.trim()
)
: ["FLAT", "HOUSE"];
const rentalIgnoredUsernames = [];
const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
.map(categoryName =>
AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
)
.filter(category => !!category);
module.exports = {
RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
RENTAL_MAX_RESULTS_PER_PAGE:
parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
RENTAL_DELAY_BETWEEN_PAGES:
parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000
};

0
app/crawler/specific/olx.js → app/crawler/specificCrawlers/olx.js
View File

0
app/crawler/specific/rental.js → app/crawler/specificCrawlers/rental.js
View File

2
test/olxScrapeTest.js
View File

@@ -1,6 +1,6 @@
"use strict";
const olxCrawler = require("../app/crawler/specific/olx");
const olxCrawler = require("../app/crawler/specificCrawlers/olx");
const urlToScrape = process.argv[2] || undefined;