30 lines
837 B
JavaScript
30 lines
837 B
JavaScript
const fetch = require('node-fetch');
|
|
const cheerio = require('cheerio');
|
|
|
|
export default class OlxCrawler {
|
|
|
|
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
|
|
this.fromPage = fromPage;
|
|
this.toPage = toPage;
|
|
this.maxResults = maxResults;
|
|
}
|
|
|
|
async sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async indexPages(start, end, maxResults = 1000) {
|
|
let results = {};
|
|
for (let i = start; i <= end; i++) {
|
|
let result = await this.indexPage(i, maxResults);
|
|
Object.assign(results, result)
|
|
await this.sleep(5000);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
async crawl() {
|
|
let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
|
|
return results;
|
|
}
|
|
} |