senaduka/old-kivi
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Rental crawler

Browse Source
This commit is contained in:
GotPPay
2017-10-16 11:36:21 +02:00
parent b2787ebda5
commit 1743171cfd
3 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawler/crawl.js
View File

@@ -14,13 +14,15 @@ import {
import 'dotenv/config';
import OlxCrawler from './specific/olx';
import ProstorCrawler from './specific/prostor';
import RentalCrawler from './specific/rental';
import MongoSaver from './savers/mongo'
install(); // for source maps to work
let crawlers = [
//new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
new ProstorCrawler(parseInt(process.env.PROSTOR_FROM_PAGE), parseInt(process.env.PROSTOR_TO_PAGE), parseInt(process.env.PROSTOR_MAX_RESULTS))
//new ProstorCrawler(parseInt(process.env.PROSTOR_FROM_PAGE), parseInt(process.env.PROSTOR_TO_PAGE), parseInt(process.env.PROSTOR_MAX_RESULTS)),
new RentalCrawler(parseInt(process.env.RENTAL_FROM_PAGE), parseInt(process.env.RENTAL_TO_PAGE), parseInt(process.env.RENTAL_MAX_RESULTS))
];
let savers = [

2
crawler/enums.js
View File

@@ -7,4 +7,6 @@ export const CATEGORY_FLAT = 0;
export const CATEGORY_HOUSE = 1;
export const CATEGORY_OFFICE = 2;
export const CATEGORY_LAND = 3;
export const CATEGORY_APARTMENT = 4;
export const CATEGORY_GARAGE = 5;

1
crawler/package.json
View File

@@ -22,6 +22,7 @@
"twilio": "^2.11.0"
},
"devDependencies": {
"babel-preset-es2015": "^6.24.1",
"webpack": "^1.13.3"
},
"scripts": {