49 lines
1.0 KiB
JavaScript
49 lines
1.0 KiB
JavaScript
'use strict'
|
|
|
|
|
|
/*
|
|
Entry point for crawling functionality
|
|
All communication between crawlers and savers is here
|
|
All environment specific configuration is read here and
|
|
passed to the crawlers and savers.
|
|
|
|
*/
|
|
import {
|
|
install
|
|
} from 'source-map-support';
|
|
import 'dotenv/config';
|
|
import OlxCrawler from './specific/olx';
|
|
import MongoSaver from './savers/mongo'
|
|
|
|
install(); // for source maps to work
|
|
|
|
let crawlers = [
|
|
new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS)
|
|
];
|
|
let savers = [
|
|
new MongoSaver(process.env.MONGO_URL)
|
|
];
|
|
|
|
let done = 0;
|
|
for (let crawler of crawlers) {
|
|
try {
|
|
crawler.crawl().then((results) => {
|
|
for (let saver of savers) {
|
|
try {
|
|
saver.open();
|
|
saver.save(results);
|
|
} catch (e) {
|
|
console.log("Error saving. Trying next saver! ", e);
|
|
|
|
}
|
|
}
|
|
});
|
|
} catch (e) {
|
|
console.log("Error crawling. Trying next crawler! ", e);
|
|
}
|
|
}
|
|
|
|
for (let saver of savers) {
|
|
saver.close();
|
|
}
|