diff --git a/app/crawler/specificCrawlers/olx.js b/app/crawler/specificCrawlers/olx.js index 87f0957..31004fe 100644 --- a/app/crawler/specificCrawlers/olx.js +++ b/app/crawler/specificCrawlers/olx.js @@ -178,32 +178,36 @@ class OlxCrawler { } async *categoryIndexer(adCategory) { - let pageToIndex = 1; + try { + let pageToIndex = 1; - const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes]; - const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory]; - if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) { - while (true) { - const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`; - const singlePageResults = await this.indexSinglePage( - urlPageToCrawl, - this.maxResultsPerPage - ); + const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes]; + const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory]; + if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) { + while (true) { + const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`; + const singlePageResults = await this.indexSinglePage( + urlPageToCrawl, + this.maxResultsPerPage + ); - await this.sleep(this.delayBetweenPages); - if (Array.isArray(singlePageResults) && singlePageResults.length > 0) { - yield singlePageResults; - } else { - return undefined; - } - - ++pageToIndex; - if (pageToIndex === this.maxPages) { - return undefined; + await this.sleep(this.delayBetweenPages); + if (Array.isArray(singlePageResults) && singlePageResults.length > 0) { + yield singlePageResults; + } else { + return undefined; + } + + ++pageToIndex; + if (pageToIndex === this.maxPages) { + return undefined; + } } + } else { + return undefined; } - } else { - return undefined; + } catch (e) { + console.log('Error inside generator: ', e); } }