36 lines
850 B
JavaScript
36 lines
850 B
JavaScript
"use strict";
|
|
/*
|
|
Entry point for crawling functionality
|
|
All communication between crawlers and savers is here
|
|
All environment specific configuration is read here and
|
|
passed to the crawlers and savers.
|
|
*/
|
|
|
|
require("dotenv").config();
|
|
const OlxCrawler = require("./specific/olx");
|
|
const { OLX_CONFIG } = require("./crawlerConfig");
|
|
const PostgresSaver = require("./savers/postgres");
|
|
|
|
const crawlers = [
|
|
new OlxCrawler(
|
|
[new PostgresSaver()],
|
|
OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
|
|
OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
|
|
OLX_CONFIG.OLX_MAX_PAGES,
|
|
OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
|
|
OLX_CONFIG.OLX_MAX_AGE
|
|
)
|
|
];
|
|
|
|
async function crawlAll() {
|
|
for (let crawler of crawlers) {
|
|
try {
|
|
await crawler.crawl();
|
|
} catch (e) {
|
|
console.log("Error crawling. Trying next crawler! ", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
crawlAll();
|