41 lines
1.0 KiB
JavaScript
41 lines
1.0 KiB
JavaScript
"use strict";
|
|
/*
|
|
Entry point for crawling functionality
|
|
All communication between crawlers and savers is here
|
|
All environment specific configuration is read here and
|
|
passed to the crawlers and savers.
|
|
*/
|
|
|
|
require("dotenv").config();
|
|
const OlxCrawler = require("./specific/olx");
|
|
const { OLX_CONFIG } = require("./crawlerConfig");
|
|
const PostgresSaver = require("./savers/postgres");
|
|
|
|
const crawlers = [
|
|
new OlxCrawler(
|
|
[new PostgresSaver()],
|
|
OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
|
|
OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
|
|
OLX_CONFIG.OLX_MAX_PAGES,
|
|
OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
|
|
OLX_CONFIG.OLX_IGNORED_USERNAMES,
|
|
OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
|
|
)
|
|
];
|
|
|
|
async function crawlAll() {
|
|
for (let crawler of crawlers) {
|
|
try {
|
|
const newRealEstates = await crawler.crawl();
|
|
|
|
console.log("Number of new real estates : ", newRealEstates.length);
|
|
} catch (e) {
|
|
console.log("Error crawling. Trying next crawler! ", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
(async () => {
|
|
await crawlAll();
|
|
})();
|