senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Add timeout to fetch wrapper

Browse Source
This commit is contained in:
=
2020-09-15 01:27:20 -07:00
parent ade3eb307d
commit 698acb010a
4 changed files with 28 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/crawler/specificCrawlers/olx.js
View File

@@ -220,6 +220,7 @@ class OlxCrawler {
const res = await fetch(url);
logDebug("Got category results for: ", url);
const body = await res.text();
logDebug("Got category results text for: ", url);
const $ = cheerio.load(body);
let hrefs = [];
@@ -260,7 +261,7 @@ class OlxCrawler {
return filteredScrapedData;
} catch (e) {
console.error("Exception caught:" + e);
console.error("Exception caught, index single page: " + e);
return [];
}
}
@@ -709,7 +710,7 @@ class OlxCrawler {
return data;
} catch (e) {
console.error("Exception caught: " + e.message, "\r\nURL:", url);
console.error("Exception caught scrapeAd : " + e.message, "\r\nURL:", url);
}
return null;
}