senaduka/old-domene-svedska
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new code fix

Browse Source
This commit is contained in:
GotPPay
2017-12-14 02:45:51 +01:00
parent a0e2e0a057
commit b4a54e84e7
6 changed files with 63131 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
crawler/crawler.js
View File

@@ -30,32 +30,26 @@ MongoClient.connect(config.databaseURL).then(mongoDatabase => {
let getDomainListPromises = [];
config.domainList.map(url=>{
let p = links.getDomainList(url).then(res=>{
console.log("One promise done");
let p2 = database.insertTodayDomains(db,res).then(()=>{
let p3 = database.getYesterdayDomains(db).then((result)=>{
getDomainListPromises.push(links.getDomainList(url).then(res=>{
return database.insertTodayDomains(db,res);
}).then(()=>{
return database.getYesterdayDomains(db);
}).then(result=>{
return database.insertExpired(db,result);
}).catch(e=>{
console.log("Err : " +e);
}));
result.map((domain)=>{
getDomainListPromises.push(database.cleanExpired(db,domain));
});
}).catch((e)=>{
console.log(e);
});
getDomainListPromises.push(p3);
});
getDomainListPromises.push(p2);
});
getDomainListPromises.push(p);
});
Promise.all(getDomainListPromises).then(()=>{
console.log("All promises done");
database.getExpiredDomains(db).then((result)=>{
links.checkExpiredDomains(db,result).then(()=>{
db.close();
});
database.getExpiredDomains(db).then(result=>{
return links.checkExpiredDomains(db,result);
}).then(()=>{
console.log("Done - closing");
db.close();
});
});
}
});