crawler/crawler.js

@@ -4,8 +4,6 @@ const database = require('./helper/database');
 var MongoClient = require ('mongodb').MongoClient;
 var ObjectID = require ('mongodb').ObjectID;
 var fs = require('fs');
-var datetime = require('node-datetime');
-var http = require('http');
 
 
 MongoClient.connect(config.databaseURL).then(mongoDatabase => {
@@ -28,33 +26,36 @@ MongoClient.connect(config.databaseURL).then(mongoDatabase => {
             config.words = config.words.map(word=>{
                 return word.toLowerCase();
             });
-
             
+            let getDomainListPromises = [];
+
             config.domainList.map(url=>{
-                links.getDomainList(url).then(res=>{
-                    database.insertTodayDomains(db,res).then(()=>{
-                        database.getYesterdayDomains(db).then((result)=>{
+                let p = links.getDomainList(url).then(res=>{
+                    console.log("One promise done");
+                    let p2 = database.insertTodayDomains(db,res).then(()=>{
+                        let p3 = database.getYesterdayDomains(db).then((result)=>{
 
                             result.map((domain)=>{
-                                db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
-                                    if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
-                                        db.collection('expired_list').insert(domain);
-                                    }
-                                });
-                            });
-
-                            database.getExpiredDomains(db).then((result)=>{
-                                links.checkExpiredDomains(db,result).then(()=>{
-                                    console.log("Done");
-                                    db.close();
-                                });
+                                getDomainListPromises.push(database.cleanExpired(db,domain));
                             });
 
                         }).catch((e)=>{
                             console.log(e);
                         });
+                        getDomainListPromises.push(p3);
                     });
-                });                
+                    getDomainListPromises.push(p2);
+                });
+                getDomainListPromises.push(p);
+            });
+
+            Promise.all(getDomainListPromises).then(()=>{
+                console.log("All promises done");
+                database.getExpiredDomains(db).then((result)=>{
+                    links.checkExpiredDomains(db,result).then(()=>{
+                        db.close();
+                    });
+                });
             });
         }
     });

crawler/helper/database.js

@@ -1,3 +1,5 @@
+var datetime = require('node-datetime');
+
 module.exports = {
     insertTodayDomains : function (db, domains){
         return new Promise((resolve,reject)=>{
@@ -13,6 +15,21 @@ module.exports = {
 
     getExpiredDomains : function (db){
         return getArrayFromDatabase(db, 'expired_list');
+    },
+
+    cleanExpired : function (db, domain){
+        return new Promise((resolve,reject)=>{
+            db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
+                if (err){
+                    reject(err);
+                }else{
+                    if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
+                        db.collection('expired_list').insert(domain);
+                    }
+                    resolve();
+                }
+            });
+        });
     }
 
 

crawler/helper/links.js

@@ -2,6 +2,7 @@ const config = require('../config');
 const https = require("https");
 const punycode = require('punycode');
 var fs = require('fs');
+var http = require('http');
 
 module.exports = {
     getDomainList : function(url){

web/src/components/DomainList.js

@@ -19,25 +19,28 @@ class DomainList extends Component {
 
   render() {
     return (
-      <ReactTable
-        data = {this.state.domains}
-        columns={[
-          {
-            Header: 'Domain name',
-            accessor: 'domainName',
-          },
-          {
-            Header: 'TLD domain',
-            accessor: 'tld',
-          },
-          {
-            Header: 'Expiration date',
-            accessor: 'expirationDate',
-          }
-        ]}
-        defaultPageSize={10}
-        className="-striped -highlight"
-      />
+      <div>
+        <ReactTable
+          data = {this.state.domains}
+          columns={[
+            {
+              Header: 'Domain name',
+              accessor: 'domainName',
+            },
+            {
+              Header: 'TLD domain',
+              accessor: 'tld',
+            },
+            {
+              Header: 'Expiration date',
+              accessor: 'expirationDate',
+            }
+          ]}
+          defaultPageSize={10}
+          className="-striped -highlight"
+        />
+        <h3> Total : {this.state.domains.length} </h3>
+      </div>
     )
   }
 }