From c9b8c2e1a5b37643349befe17988513ad598df62 Mon Sep 17 00:00:00 2001
From: = <edin.dazdarevic@gmail.com>
Date: Thu, 10 Sep 2020 13:31:57 -0700
Subject: [PATCH] Fiks

---
 app/crawler/specificCrawlers/olx.js | 6 ++++--
 package-lock.json                   | 4 +---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/crawler/specificCrawlers/olx.js b/app/crawler/specificCrawlers/olx.js
index a979089..405a613 100644
--- a/app/crawler/specificCrawlers/olx.js
+++ b/app/crawler/specificCrawlers/olx.js
@@ -178,6 +178,7 @@ class OlxCrawler {
           this.maxResultsPerPage
         );
 
+        await this.sleep(this.delayBetweenPages);
         if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
           yield singlePageResults;
         } else {
@@ -227,10 +228,11 @@ class OlxCrawler {
 
 
       const { scrapedData, errors } = await PromisePool
-        .withConcurrency(3)
+        .withConcurrency(2)
         .for(asyncScraping)
         .process(async data => {
           const result = await this.scrapeAd(data)
+          await this.sleep(this.delayBetweenPages);
           return result;
         })
 
@@ -243,7 +245,7 @@ class OlxCrawler {
   }
 
   async scrapeAd(url) {
-    // console.log("Scraping : ", url);
+    //console.log("Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
diff --git a/package-lock.json b/package-lock.json
index f45e4f8..7bee37e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -43,14 +43,12 @@
     "@supercharge/goodies": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz",
-      "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g==",
-      "dev": true
+      "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g=="
     },
     "@supercharge/promise-pool": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz",
       "integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==",
-      "dev": true,
       "requires": {
         "@supercharge/goodies": "~1.4.0"
       }