senaduka/old-domene-svedska
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

error fix

Browse Source
This commit is contained in:
GotPPay
2017-12-13 13:56:47 +01:00
parent 7fd880ba1c
commit a0e2e0a057
4 changed files with 60 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
crawler/crawler.js
View File

@@ -4,8 +4,6 @@ const database = require('./helper/database');
var MongoClient = require ('mongodb').MongoClient;
var ObjectID = require ('mongodb').ObjectID;
var fs = require('fs');
var datetime = require('node-datetime');
var http = require('http');
MongoClient.connect(config.databaseURL).then(mongoDatabase => {
@@ -28,33 +26,36 @@ MongoClient.connect(config.databaseURL).then(mongoDatabase => {
config.words = config.words.map(word=>{
return word.toLowerCase();
});
let getDomainListPromises = [];
config.domainList.map(url=>{
links.getDomainList(url).then(res=>{
database.insertTodayDomains(db,res).then(()=>{
database.getYesterdayDomains(db).then((result)=>{
let p = links.getDomainList(url).then(res=>{
console.log("One promise done");
let p2 = database.insertTodayDomains(db,res).then(()=>{
let p3 = database.getYesterdayDomains(db).then((result)=>{
result.map((domain)=>{
db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
db.collection('expired_list').insert(domain);
}
});
});
database.getExpiredDomains(db).then((result)=>{
links.checkExpiredDomains(db,result).then(()=>{
console.log("Done");
db.close();
});
getDomainListPromises.push(database.cleanExpired(db,domain));
});
}).catch((e)=>{
console.log(e);
});
getDomainListPromises.push(p3);
});
});
getDomainListPromises.push(p2);
});
getDomainListPromises.push(p);
});
Promise.all(getDomainListPromises).then(()=>{
console.log("All promises done");
database.getExpiredDomains(db).then((result)=>{
links.checkExpiredDomains(db,result).then(()=>{
db.close();
});
});
});
}
});