Added hrefs to global varialbe

app/services/crawlerService.js

@@ -20,7 +20,22 @@ async function crawlAll() {
         try {
 
             const marketAlertsFromDb = await allMarketAlerts(true);
-            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
+            const hrefs = [];
+            const subscribedMakretAlerts = marketAlertsFromDb.filter(marketAlert => {
+                return marketAlert.subscribed;
+            });
+            marketAlertsFromDb.map(marketAlert => {
+                if (hrefs[marketAlert.request] === undefined) {
+                    hrefs[marketAlert.request] = [] 
+                }
+
+                hrefs[marketAlert.request].push(marketAlert.url);
+            })
+
+            global.hrefs = hrefs;
+            console.log(global.hrefs);
+            
+            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + subscribedMakretAlerts.length);
 
             const marketAlerts = [];
             const mergedResults = [].concat.apply([], results);
@@ -47,7 +62,7 @@ async function crawlAll() {
 
             try {
 
-                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+                const filteredMarketAlerts = marketAlerts.filter((elem) => !subscribedMakretAlerts.find(({ url }) => elem.url === url));
                 console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
 
                 await db.MarketAlert.bulkCreate(filteredMarketAlerts);