Add timeout to fetch wrapper

app/crawler/specificCrawlers/olx.js

@@ -220,6 +220,7 @@ class OlxCrawler {
       const res = await fetch(url);
       logDebug("Got category results for: ", url);
       const body = await res.text();
+      logDebug("Got category results text for: ", url);
       const $ = cheerio.load(body);
       let hrefs = [];
 
@@ -260,7 +261,7 @@ class OlxCrawler {
 
       return filteredScrapedData;
     } catch (e) {
-      console.error("Exception caught:" + e);
+      console.error("Exception caught, index single page: " + e);
       return [];
     }
   }
@@ -709,7 +710,7 @@ class OlxCrawler {
 
       return data;
     } catch (e) {
-      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+      console.error("Exception caught scrapeAd : " + e.message, "\r\nURL:", url);
     }
     return null;
   }

app/helpers/fetchWrapper.js

@@ -1,4 +1,5 @@
 const nodeFetch = require("node-fetch");
+const AbortController = require('abort-controller');
 const {
   USER_AGENT,
   USE_SCRAPER_API,
@@ -11,10 +12,15 @@ const timeout = (ms) => {
 }
 
 const fetch = async (url, options = {}) => {
+  const controller = new AbortController();
+
   const newOptions = Object.assign({}, options);
   if (!newOptions["headers"]) {
     newOptions["headers"] = {};
   }
+
+  newOptions.signal = controller.signal;
+
  // newOptions["headers"]["User-Agent"] = USER_AGENT;
 
   let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
@@ -25,7 +31,9 @@ const fetch = async (url, options = {}) => {
   const urlAdaptedForScraping = USE_SCRAPER_API
     ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
     : url;
-  return nodeFetch(urlAdaptedForScraping, newOptions);
+  const result = nodeFetch(urlAdaptedForScraping, newOptions);
+  const timeoutId = setTimeout(() => controller.abort(), 5000);
+  return result;
 };
 
 module.exports = fetch;