handle failed page fetch

2020-05-18 03:53:08 +02:00
parent 65068932ad
commit 159fedbc2d
4 changed files with 17 additions and 1 deletions
--- a/app/crawler/specificCrawlers/saljic.js
+++ b/app/crawler/specificCrawlers/saljic.js
@@ -220,12 +220,16 @@ class SaljicCrawler {
  }

  async scrapeAd(url, adType) {
-    console.log("[SALJIC] Scraping : ", url);
+    // console.log("[SALJIC] Scraping : ", url);
    try {
      const adPageSource = await fetch(url);
      const body = await adPageSource.text();
      const $ = cheerio.load(body);

+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
      // No information for status ex. PRODAN
      const status = AD_STATUS.STATUS_NORMAL;
      //Extracting agency ID from url