senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Add try/catch to generator

Browse Source
This commit is contained in:
=
2020-09-14 14:54:39 -07:00
parent 8031f6f8a3
commit ade3eb307d
1 changed files with 26 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
app/crawler/specificCrawlers/olx.js
View File

@@ -178,32 +178,36 @@ class OlxCrawler {
} }
async *categoryIndexer(adCategory) { async *categoryIndexer(adCategory) {
let pageToIndex = 1; try {
let pageToIndex = 1;
const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes]; const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory]; const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) { if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
while (true) { while (true) {
const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`; const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
const singlePageResults = await this.indexSinglePage( const singlePageResults = await this.indexSinglePage(
urlPageToCrawl, urlPageToCrawl,
this.maxResultsPerPage this.maxResultsPerPage
); );
await this.sleep(this.delayBetweenPages); await this.sleep(this.delayBetweenPages);
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) { if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
yield singlePageResults; yield singlePageResults;
} else { } else {
return undefined; return undefined;
} }
++pageToIndex; ++pageToIndex;
if (pageToIndex === this.maxPages) { if (pageToIndex === this.maxPages) {
return undefined; return undefined;
}
} }
} else {
return undefined;
} }
} else { } catch (e) {
return undefined; console.log('Error inside generator: ', e);
} }
} }