senaduka/old-kivi
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Prostor.ba crawler

Browse Source
This commit is contained in:
Edin Dazdarevic
2017-04-10 05:28:37 +02:00
parent 738720aa13
commit a1151150db
6 changed files with 262 additions and 25024 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crawler/crawl.js
View File

@@ -13,13 +13,16 @@ import {
} from 'source-map-support';
import 'dotenv/config';
import OlxCrawler from './specific/olx';
import ProstorCrawler from './specific/prostor';
import MongoSaver from './savers/mongo'
install(); // for source maps to work
let crawlers = [
new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS)
//new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
new ProstorCrawler(process.env.PROSTOR_FROM_PAGE, process.env.PROSTOR_TO_PAGE, process.env.PROSTOR_MAX_RESULTS)
];
let savers = [
new MongoSaver(process.env.MONGO_URL)
];