senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

add crawler and crawler config for Aktido agency

Browse Source
This commit is contained in:
Bilal Catic
2019-10-28 09:14:45 +01:00
parent f5d912f02c
commit f899c96dc6
6 changed files with 438 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
app/crawler/crawl.js
View File

@@ -8,11 +8,13 @@
const OlxCrawler = require("./specificCrawlers/olx");
const RentalCrawler = require("./specificCrawlers/rental");
const ProstorCrawler = require("./specificCrawlers/prostor");
const AktidoCrawler = require("./specificCrawlers/aktido");
const {
OLX_CONFIG,
RENTAL_CONFIG,
PROSTOR_CONFIG
PROSTOR_CONFIG,
AKTIDO_CONFIG
} = require("./crawlerConfig");
const PostgresSaver = require("./savers/postgres");
@@ -46,6 +48,15 @@ async function crawlAll() {
PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
),
new AktidoCrawler(
[postgresSaver],
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
)
];