crawler/crawler.js

@@ -4,8 +4,6 @@ const database = require('./helper/database');
 var MongoClient = require ('mongodb').MongoClient;
 var ObjectID = require ('mongodb').ObjectID;
 var fs = require('fs');
-var datetime = require('node-datetime');
-var http = require('http');
 
 
 MongoClient.connect(config.databaseURL).then(mongoDatabase => {
@@ -29,30 +27,33 @@ MongoClient.connect(config.databaseURL).then(mongoDatabase => {
                 return word.toLowerCase();
             });
             
+            let getDomainListPromises = [];
 
             config.domainList.map(url=>{
-                links.getDomainList(url).then(res=>{
-                    database.insertTodayDomains(db,res).then(()=>{
-                        database.getYesterdayDomains(db).then((result)=>{
+                let p = links.getDomainList(url).then(res=>{
+                    console.log("One promise done");
+                    let p2 = database.insertTodayDomains(db,res).then(()=>{
+                        let p3 = database.getYesterdayDomains(db).then((result)=>{
 
                             result.map((domain)=>{
-                                db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
-                                    if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
-                                        db.collection('expired_list').insert(domain);
-                                    }
-                                });
-                            });
-
-                            database.getExpiredDomains(db).then((result)=>{
-                                links.checkExpiredDomains(db,result).then(()=>{
-                                    console.log("Done");
-                                    db.close();
-                                });
+                                getDomainListPromises.push(database.cleanExpired(db,domain));
                             });
 
                         }).catch((e)=>{
                             console.log(e);
                         });
+                        getDomainListPromises.push(p3);
+                    });
+                    getDomainListPromises.push(p2);
+                });
+                getDomainListPromises.push(p);
+            });
+
+            Promise.all(getDomainListPromises).then(()=>{
+                console.log("All promises done");
+                database.getExpiredDomains(db).then((result)=>{
+                    links.checkExpiredDomains(db,result).then(()=>{
+                        db.close();
                     });
                 });
             });

crawler/helper/database.js

@@ -1,3 +1,5 @@
+var datetime = require('node-datetime');
+
 module.exports = {
     insertTodayDomains : function (db, domains){
         return new Promise((resolve,reject)=>{
@@ -13,6 +15,21 @@ module.exports = {
 
     getExpiredDomains : function (db){
         return getArrayFromDatabase(db, 'expired_list');
+    },
+
+    cleanExpired : function (db, domain){
+        return new Promise((resolve,reject)=>{
+            db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
+                if (err){
+                    reject(err);
+                }else{
+                    if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
+                        db.collection('expired_list').insert(domain);
+                    }
+                    resolve();
+                }
+            });
+        });
     }
 
 

crawler/helper/links.js

@@ -2,6 +2,7 @@ const config = require('../config');
 const https = require("https");
 const punycode = require('punycode');
 var fs = require('fs');
+var http = require('http');
 
 module.exports = {
     getDomainList : function(url){

web/src/components/DomainList.js

@@ -19,6 +19,7 @@ class DomainList extends Component {
 
   render() {
     return (
+      <div>
         <ReactTable
           data = {this.state.domains}
           columns={[
@@ -38,6 +39,8 @@ class DomainList extends Component {
           defaultPageSize={10}
           className="-striped -highlight"
         />
+        <h3> Total : {this.state.domains.length} </h3>
+      </div>
     )
   }
 }