handle failed page fetch

2020-05-18 03:53:08 +02:00
parent 65068932ad
commit 159fedbc2d
4 changed files with 17 additions and 1 deletions
--- a/app/crawler/specificCrawlers/prostor.js
+++ b/app/crawler/specificCrawlers/prostor.js
@@ -215,6 +215,10 @@ class ProstorCrawler {
      const body = await adPageSource.text();
      const $ = cheerio.load(body);

+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
      // link contains part of the URL in the format of : /prodaja/stan/stup/9556
      // general form is : /actionType/realEstateType/location/realEstateID
      // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']