diff --git a/crawler/crawler.js b/crawler/crawler.js index 6de7c36..63efc27 100644 --- a/crawler/crawler.js +++ b/crawler/crawler.js @@ -4,8 +4,6 @@ const database = require('./helper/database'); var MongoClient = require ('mongodb').MongoClient; var ObjectID = require ('mongodb').ObjectID; var fs = require('fs'); -var datetime = require('node-datetime'); -var http = require('http'); MongoClient.connect(config.databaseURL).then(mongoDatabase => { @@ -28,33 +26,36 @@ MongoClient.connect(config.databaseURL).then(mongoDatabase => { config.words = config.words.map(word=>{ return word.toLowerCase(); }); - + let getDomainListPromises = []; + config.domainList.map(url=>{ - links.getDomainList(url).then(res=>{ - database.insertTodayDomains(db,res).then(()=>{ - database.getYesterdayDomains(db).then((result)=>{ + let p = links.getDomainList(url).then(res=>{ + console.log("One promise done"); + let p2 = database.insertTodayDomains(db,res).then(()=>{ + let p3 = database.getYesterdayDomains(db).then((result)=>{ result.map((domain)=>{ - db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{ - if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){ - db.collection('expired_list').insert(domain); - } - }); - }); - - database.getExpiredDomains(db).then((result)=>{ - links.checkExpiredDomains(db,result).then(()=>{ - console.log("Done"); - db.close(); - }); + getDomainListPromises.push(database.cleanExpired(db,domain)); }); }).catch((e)=>{ console.log(e); }); + getDomainListPromises.push(p3); }); - }); + getDomainListPromises.push(p2); + }); + getDomainListPromises.push(p); + }); + + Promise.all(getDomainListPromises).then(()=>{ + console.log("All promises done"); + database.getExpiredDomains(db).then((result)=>{ + links.checkExpiredDomains(db,result).then(()=>{ + db.close(); + }); + }); }); } }); diff --git a/crawler/helper/database.js b/crawler/helper/database.js index 88be460..641782a 100644 --- a/crawler/helper/database.js +++ b/crawler/helper/database.js @@ -1,3 +1,5 @@ +var datetime = require('node-datetime'); + module.exports = { insertTodayDomains : function (db, domains){ return new Promise((resolve,reject)=>{ @@ -13,6 +15,21 @@ module.exports = { getExpiredDomains : function (db){ return getArrayFromDatabase(db, 'expired_list'); + }, + + cleanExpired : function (db, domain){ + return new Promise((resolve,reject)=>{ + db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{ + if (err){ + reject(err); + }else{ + if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){ + db.collection('expired_list').insert(domain); + } + resolve(); + } + }); + }); } diff --git a/crawler/helper/links.js b/crawler/helper/links.js index fd933de..ba1f9e6 100644 --- a/crawler/helper/links.js +++ b/crawler/helper/links.js @@ -2,6 +2,7 @@ const config = require('../config'); const https = require("https"); const punycode = require('punycode'); var fs = require('fs'); +var http = require('http'); module.exports = { getDomainList : function(url){ diff --git a/web/src/components/DomainList.js b/web/src/components/DomainList.js index 148572d..c166c80 100644 --- a/web/src/components/DomainList.js +++ b/web/src/components/DomainList.js @@ -19,25 +19,28 @@ class DomainList extends Component { render() { return ( - +
+ +

Total : {this.state.domains.length}

+
) } }