Promise pool

app/crawler/specificCrawlers/olx.js

@@ -4,6 +4,7 @@ const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
+const PromisePool = require('@supercharge/promise-pool'); 
 
 const {
   AD_TYPE,
@@ -224,6 +225,14 @@ class OlxCrawler {
         asyncScraping.push(this.scrapeAd(hrefs[i]));
       }
 
+
+      const { scrapedData, errors } = await PromisePool
+        .withConcurrency(2)
+        .for(asyncScraping)
+        .process(async data => {
+          return await data
+      })
+
       const scrapedData = await Promise.all(asyncScraping);
       const filteredScrapedData = scrapedData.filter(adData => !!adData);
       return filteredScrapedData;

app/helpers/fetchWrapper.js

@@ -25,8 +25,6 @@ const fetch = async (url, options = {}) => {
   const urlAdaptedForScraping = USE_SCRAPER_API
     ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
     : url;
-  const randomPauseMS = Math.floor(Math.random() * Math.floor(1500)) + 500;
-  await timeout(randomPauseMS);
   return nodeFetch(urlAdaptedForScraping, newOptions);
 };