Created non-concurrent request groups.

2020-03-03 15:21:51 +01:00
parent ccea5fe2aa
commit f5f8fa276c
5 changed files with 60 additions and 19 deletions
--- a/app/crawler/specificConfigs/olx.js
+++ b/app/crawler/specificConfigs/olx.js
@@ -35,6 +35,5 @@ module.exports = {
  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
  OLX_DELAY_BETWEEN_PAGES:
    parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000,
-  OLX_DELAY_BETWEEN_ADS: parseInt(process.env.OLX_DELAY_BETWEEN_ADS) || 1000,
  OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL)
 };