From d3b2f95719c74a06e224f349048d327a5ff14ccd Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Wed, 18 Sep 2019 15:33:09 +0200 Subject: [PATCH] add new ENV variables to control crawler --- development.env | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/development.env b/development.env index 47d75c3..f998e9d 100644 --- a/development.env +++ b/development.env @@ -13,3 +13,11 @@ AMAZON_SECRET_ACCESS_KEY=(your-key-here) AMAZON_REGION=eu-west-1 APP_URL=http://localhost:3001 SOURCE_EMAIL=info@saburly.com + +#=============== CRAWLER SETTINGS===============# +#==OLX== +OLX_START_PAGE=Crawler starts from this page +OLX_END_PAGE=Crawler ends with this page (including this page) +OLX_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved +OLX_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values +OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values