diff --git a/crawler/crawler.js b/crawler/crawler.js
index 6de7c36..63efc27 100644
--- a/crawler/crawler.js
+++ b/crawler/crawler.js
@@ -4,8 +4,6 @@ const database = require('./helper/database');
var MongoClient = require ('mongodb').MongoClient;
var ObjectID = require ('mongodb').ObjectID;
var fs = require('fs');
-var datetime = require('node-datetime');
-var http = require('http');
MongoClient.connect(config.databaseURL).then(mongoDatabase => {
@@ -28,33 +26,36 @@ MongoClient.connect(config.databaseURL).then(mongoDatabase => {
config.words = config.words.map(word=>{
return word.toLowerCase();
});
-
+ let getDomainListPromises = [];
+
config.domainList.map(url=>{
- links.getDomainList(url).then(res=>{
- database.insertTodayDomains(db,res).then(()=>{
- database.getYesterdayDomains(db).then((result)=>{
+ let p = links.getDomainList(url).then(res=>{
+ console.log("One promise done");
+ let p2 = database.insertTodayDomains(db,res).then(()=>{
+ let p3 = database.getYesterdayDomains(db).then((result)=>{
result.map((domain)=>{
- db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
- if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
- db.collection('expired_list').insert(domain);
- }
- });
- });
-
- database.getExpiredDomains(db).then((result)=>{
- links.checkExpiredDomains(db,result).then(()=>{
- console.log("Done");
- db.close();
- });
+ getDomainListPromises.push(database.cleanExpired(db,domain));
});
}).catch((e)=>{
console.log(e);
});
+ getDomainListPromises.push(p3);
});
- });
+ getDomainListPromises.push(p2);
+ });
+ getDomainListPromises.push(p);
+ });
+
+ Promise.all(getDomainListPromises).then(()=>{
+ console.log("All promises done");
+ database.getExpiredDomains(db).then((result)=>{
+ links.checkExpiredDomains(db,result).then(()=>{
+ db.close();
+ });
+ });
});
}
});
diff --git a/crawler/helper/database.js b/crawler/helper/database.js
index 88be460..641782a 100644
--- a/crawler/helper/database.js
+++ b/crawler/helper/database.js
@@ -1,3 +1,5 @@
+var datetime = require('node-datetime');
+
module.exports = {
insertTodayDomains : function (db, domains){
return new Promise((resolve,reject)=>{
@@ -13,6 +15,21 @@ module.exports = {
getExpiredDomains : function (db){
return getArrayFromDatabase(db, 'expired_list');
+ },
+
+ cleanExpired : function (db, domain){
+ return new Promise((resolve,reject)=>{
+ db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
+ if (err){
+ reject(err);
+ }else{
+ if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
+ db.collection('expired_list').insert(domain);
+ }
+ resolve();
+ }
+ });
+ });
}
diff --git a/crawler/helper/links.js b/crawler/helper/links.js
index fd933de..ba1f9e6 100644
--- a/crawler/helper/links.js
+++ b/crawler/helper/links.js
@@ -2,6 +2,7 @@ const config = require('../config');
const https = require("https");
const punycode = require('punycode');
var fs = require('fs');
+var http = require('http');
module.exports = {
getDomainList : function(url){
diff --git a/web/src/components/DomainList.js b/web/src/components/DomainList.js
index 148572d..c166c80 100644
--- a/web/src/components/DomainList.js
+++ b/web/src/components/DomainList.js
@@ -19,25 +19,28 @@ class DomainList extends Component {
render() {
return (
-