From 159fedbc2d9d76d7619e46b2f8f2b96e59576774 Mon Sep 17 00:00:00 2001
From: Bilal <bilal@saburly.com>
Date: Mon, 18 May 2020 03:53:08 +0200
Subject: [PATCH] handle failed page fetch

---
 app/crawler/specificCrawlers/aktido.js  | 4 ++++
 app/crawler/specificCrawlers/prostor.js | 4 ++++
 app/crawler/specificCrawlers/rental.js  | 4 ++++
 app/crawler/specificCrawlers/saljic.js  | 6 +++++-
 4 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/app/crawler/specificCrawlers/aktido.js b/app/crawler/specificCrawlers/aktido.js
index 9755eb7..ac4dcc6 100644
--- a/app/crawler/specificCrawlers/aktido.js
+++ b/app/crawler/specificCrawlers/aktido.js
@@ -202,6 +202,10 @@ class AktidoCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       const mapElementParent = $(".box-map").parent();
       const scriptElement = $("script", mapElementParent);
       if (
diff --git a/app/crawler/specificCrawlers/prostor.js b/app/crawler/specificCrawlers/prostor.js
index 251c514..dd0b3c7 100644
--- a/app/crawler/specificCrawlers/prostor.js
+++ b/app/crawler/specificCrawlers/prostor.js
@@ -215,6 +215,10 @@ class ProstorCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       // link contains part of the URL in the format of : /prodaja/stan/stup/9556
       // general form is : /actionType/realEstateType/location/realEstateID
       // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
diff --git a/app/crawler/specificCrawlers/rental.js b/app/crawler/specificCrawlers/rental.js
index ed37e83..292885c 100644
--- a/app/crawler/specificCrawlers/rental.js
+++ b/app/crawler/specificCrawlers/rental.js
@@ -202,6 +202,10 @@ class RentalCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       const mapElementParent = $(".box-map").parent();
       const scriptElement = $("script", mapElementParent);
       if (
diff --git a/app/crawler/specificCrawlers/saljic.js b/app/crawler/specificCrawlers/saljic.js
index 3d5e30e..9f5b0b7 100644
--- a/app/crawler/specificCrawlers/saljic.js
+++ b/app/crawler/specificCrawlers/saljic.js
@@ -220,12 +220,16 @@ class SaljicCrawler {
   }
 
   async scrapeAd(url, adType) {
-    console.log("[SALJIC] Scraping : ", url);
+    // console.log("[SALJIC] Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       // No information for status ex. PRODAN
       const status = AD_STATUS.STATUS_NORMAL;
       //Extracting agency ID from url