Fiks

app/crawler/specificCrawlers/olx.js

@@ -178,6 +178,7 @@ class OlxCrawler {
           this.maxResultsPerPage
         );
 
+        await this.sleep(this.delayBetweenPages);
         if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
           yield singlePageResults;
         } else {
@@ -227,10 +228,11 @@ class OlxCrawler {
 
 
       const { scrapedData, errors } = await PromisePool
-        .withConcurrency(3)
+        .withConcurrency(2)
         .for(asyncScraping)
         .process(async data => {
           const result = await this.scrapeAd(data)
+          await this.sleep(this.delayBetweenPages);
           return result;
         })
 
@@ -243,7 +245,7 @@ class OlxCrawler {
   }
 
   async scrapeAd(url) {
-    // console.log("Scraping : ", url);
+    //console.log("Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();