Fiks

app/crawler/specificCrawlers/olx.js

@@ -178,6 +178,7 @@ class OlxCrawler {
           this.maxResultsPerPage
         );
 
+        await this.sleep(this.delayBetweenPages);
         if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
           yield singlePageResults;
         } else {
@@ -227,10 +228,11 @@ class OlxCrawler {
 
 
       const { scrapedData, errors } = await PromisePool
-        .withConcurrency(3)
+        .withConcurrency(2)
         .for(asyncScraping)
         .process(async data => {
           const result = await this.scrapeAd(data)
+          await this.sleep(this.delayBetweenPages);
           return result;
         })
 
@@ -243,7 +245,7 @@ class OlxCrawler {
   }
 
   async scrapeAd(url) {
-    // console.log("Scraping : ", url);
+    //console.log("Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();

package-lock.json

@@ -43,14 +43,12 @@
     "@supercharge/goodies": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz",
-      "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g==",
-      "dev": true
+      "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g=="
     },
     "@supercharge/promise-pool": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz",
       "integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==",
-      "dev": true,
       "requires": {
         "@supercharge/goodies": "~1.4.0"
       }