From ecc5b174a075d9e501c9fc2effe9f5b9dd168362 Mon Sep 17 00:00:00 2001
From: Bilal Catic <bilal.catic@hotmail.com>
Date: Wed, 30 Oct 2019 17:23:43 +0100
Subject: [PATCH] implement RENT option for Aktido; implement force crawl
 option

---
 app/crawler/specificConfigs/aktido.js  |  3 ++-
 app/crawler/specificCrawlers/aktido.js | 26 +++++++-------------------
 2 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/app/crawler/specificConfigs/aktido.js b/app/crawler/specificConfigs/aktido.js
index bd06645..092f09d 100644
--- a/app/crawler/specificConfigs/aktido.js
+++ b/app/crawler/specificConfigs/aktido.js
@@ -29,5 +29,6 @@ module.exports = {
   AKTIDO_CRAWLER_AD_CATEGORIES: transformedAktidoCrawlerAdCategories,
   AKTIDO_IGNORED_USERNAMES: aktidoIgnoredUsernames || [],
   AKTIDO_DELAY_BETWEEN_PAGES:
-    parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000
+    parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000,
+  AKTIDO_FORCE_CRAWL: !!parseInt(process.env.AKTIDO_FORCE_CRAWL)
 };
diff --git a/app/crawler/specificCrawlers/aktido.js b/app/crawler/specificCrawlers/aktido.js
index a2ea43d..6512f15 100644
--- a/app/crawler/specificCrawlers/aktido.js
+++ b/app/crawler/specificCrawlers/aktido.js
@@ -39,6 +39,8 @@ const AKTIDO_ENUMS = {
   AKTIDO_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
 };
 
+const { AKTIDO_FORCE_CRAWL } = require("../specificConfigs/aktido");
+
 class AktidoCrawler {
   constructor(
     savers = [],
@@ -88,27 +90,13 @@ class AktidoCrawler {
 
             newRealEstates.push(...newRecords);
 
-            if (Array.isArray(newRecords) && newRecords.length === 0) {
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !AKTIDO_FORCE_CRAWL
+            ) {
               generatorsToRemove[index] = true;
             }
-
-            // for (const existingRecord of existingRecords) {
-            //   const { publishedDate, renewedDate } = existingRecord;
-            //
-            //   const publishedDateMoment = moment.utc(publishedDate);
-            //   const renewedDateMoment = moment.utc(renewedDate);
-            //
-            //   const stopCrawlingThisCategory = publishedDateMoment.isSame(
-            //     renewedDateMoment,
-            //     "minute"
-            //   );
-            //
-            //   if (stopCrawlingThisCategory) {
-            //     generatorsToRemove[index] = true;
-            //     // console.log("\tGenerator ", index + 1, "has no more new ads");
-            //     break;
-            //   }
-            // }
           } else {
             //Generator returned undefined, remove this generator from array
             generatorsToRemove[index] = true;