senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

implement crawler for Prostor agency

Browse Source
This commit is contained in:
Bilal Catic
2019-10-25 10:54:08 +02:00
parent 05fad652c4
commit 7e3b0bfcd5
3 changed files with 264 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/crawler/crawl.js
View File

@@ -7,8 +7,13 @@
*/
const OlxCrawler = require("./specificCrawlers/olx");
const RentalCrawler = require("./specificCrawlers/rental");
const ProstorCrawler = require("./specificCrawlers/prostor");
const { OLX_CONFIG, RENTAL_CONFIG } = require("./crawlerConfig");
const {
OLX_CONFIG,
RENTAL_CONFIG,
PROSTOR_CONFIG
} = require("./crawlerConfig");
const PostgresSaver = require("./savers/postgres");
async function crawlAll() {
@@ -32,6 +37,15 @@ async function crawlAll() {
RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
),
new ProstorCrawler(
[postgresSaver],
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
)
];