new code fix

crawler/helper/links.js

@@ -7,18 +7,17 @@ var http = require('http');
 module.exports = {
     getDomainList : function(url){
         return new Promise((resolve, reject)=>{
+
             getRawDomainList(url).then(raw=>{
-                processDomains(raw).then(result=>{
-                    applyFilter(result).then(result=>{
-                        resolve(result);
-                    })
-                });
+                resolve(applyFilter(processDomains(raw)));
             });
         });
     },
 
     checkExpiredDomains : function(db, domains){
         return new Promise((resolve,reject)=>{
+            let waitingPromises = [];
+            let domainsForRemoval = [];
             domains.map(domain=>{
                 let checkLink = '';
                 switch(domain.tld){
@@ -31,62 +30,85 @@ module.exports = {
                 }
     
                 let fullName = domain.domainName + '.' + domain.tld;
-                http.get(checkLink+punycode.toASCII(fullName), res => {
-                    res.setEncoding("utf8");
-                    let body = "";
-                    res.on("data", data => {
-                        body += data;
-                    });
-                    res.on("end", () => {
-                        let status = body.split(' ')[0];
-                        if (status !== 'free'){
-                            db.collection('expired_list').remove({domainName:domain.domainName});
-                        }
-                    });
-                });
+                waitingPromises.push(removeOccupiedDomain(db, checkLink+punycode.toASCII(fullName),domain));
+            });
+            
+            Promise.all(waitingPromises).then(()=>{
+                resolve();
             });
-            resolve();
         });
     }
-
-    
 };
 
-var applyFilter = function (domains){
+var isDomainFree = function (db, url, domain){
     return new Promise((resolve,reject)=>{
-        //get domain names that only match whole words
-        let result = [];
-        domains.map(domain=>{
-            let index = config.words.indexOf(domain.domainName);
-            if (index !== -1){
-                result.push(domain);
-            }
+        http.get(url, res => {
+            res.setEncoding("utf8");
+            let body = "";
+            res.on("data", data => {
+                body += data;
+            });
+            res.on("end", () => {
+                let status = body.split(' ')[0];
+                if (status === 'free'){
+                    resolve(true);
+                }else{
+                    resolve(false);
+                }
+            });
         });
-        resolve(result);
     });
 }
 
-var processDomains = function(raw){
+var removeOccupiedDomain = function (db, url,domain){
     return new Promise((resolve,reject)=>{
-        let result = [];
-        raw.split('\n').map(domain=>{
-            let unicodeDomain = punycode.toUnicode(domain);
-            let dot = unicodeDomain.indexOf('.');
-            let tab = unicodeDomain.indexOf('\t');
-            if (dot !== -1){
-                let domainName = unicodeDomain.substring(0,dot);
-                let tld = unicodeDomain.substring(dot+1,tab);
-                if (domainName.match(config.swedishLettersOnly)){
-                    //domain name contains only letters
-                    //line in domain list is formatted as follows : [domain name]\t[expiration date]
-                    result.push({domainName: domainName, tld:tld ,expirationDate: domain.split('\t')[1]});
+        http.get(url, res => {
+            res.setEncoding("utf8");
+            let body = "";
+            res.on("data", data => {
+                body += data;
+            });
+            res.on("end", () => {
+                let status = body.split(' ')[0];
+                if (status !== 'free'){
+                    db.collection('expired_list').remove({domainName:domain.domainName, tld: domain.tld});
                 }
-            }
+                resolve();
+            });
         });
-        resolve(result);
     });
 }
 
+var applyFilter = function (domains){
+    let result = [];
+    domains.map(domain=>{
+        let index = config.words.indexOf(domain.domainName);
+        if (index !== -1){
+            result.push(domain);
+        }
+    });
+    return result;
+}
+
+var processDomains = function(raw){
+    let result = [];
+    raw.split('\n').map(domain=>{
+        let unicodeDomain = punycode.toUnicode(domain);
+        let dot = unicodeDomain.indexOf('.');
+        let tab = unicodeDomain.indexOf('\t');
+        if (dot !== -1){
+            let domainName = unicodeDomain.substring(0,dot);
+            let tld = unicodeDomain.substring(dot+1,tab);
+            if (domainName.match(config.swedishLettersOnly)){
+                //domain name contains only letters
+                //line in domain list is formatted as follows : [domain name]\t[expiration date]
+                result.push({domainName: domainName, tld:tld ,expirationDate: domain.split('\t')[1]});
+            }
+        }
+    });
+    return result;
+}
+
 var getRawDomainList = function (url) {
     return new Promise((resolve, reject)=>{
         if (url[0]==='/'){