33 lines
823 B
JavaScript
33 lines
823 B
JavaScript
|
|
// import OlxCrawler from '../helpers/crawlers/olixClawler'
|
|
const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
|
|
|
const crawlers = [
|
|
new OlxCrawler(1, 2, 3),
|
|
// new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
|
|
];
|
|
|
|
async function crawlAll() {
|
|
|
|
for (let crawler of crawlers) {
|
|
try {
|
|
let results = await crawler.crawl()
|
|
for (let saver of savers) {
|
|
try {
|
|
await saver.connect();
|
|
await saver.save(results);
|
|
} catch (e) {
|
|
console.log("Error saving. Trying next saver! ", e);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.log("Error crawling. Trying next crawler! ", e);
|
|
}
|
|
}
|
|
|
|
for (let saver of savers) {
|
|
saver.close();
|
|
}
|
|
}
|
|
|
|
crawlAll(); |