65 lines
2.2 KiB
JavaScript
65 lines
2.2 KiB
JavaScript
var config = require('./config');
|
|
const links = require('./helper/links');
|
|
const database = require('./helper/database');
|
|
var MongoClient = require ('mongodb').MongoClient;
|
|
var ObjectID = require ('mongodb').ObjectID;
|
|
var fs = require('fs');
|
|
|
|
|
|
MongoClient.connect(config.databaseURL).then(mongoDatabase => {
|
|
let db = mongoDatabase;
|
|
|
|
db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
|
|
db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});
|
|
|
|
db.collection('yesterday').drop();
|
|
db.collection('today').rename('yesterday');
|
|
db.createCollection('today');
|
|
|
|
//Get word list into memory
|
|
fs.readFile(config.wordList, 'utf8', (err,data)=>{
|
|
if (err){
|
|
console.log(err);
|
|
config.words=[];
|
|
}else{
|
|
config.words = data.split('\n');
|
|
config.words = config.words.map(word=>{
|
|
return word.toLowerCase();
|
|
});
|
|
|
|
let getDomainListPromises = [];
|
|
|
|
config.domainList.map(url=>{
|
|
let p = links.getDomainList(url).then(res=>{
|
|
console.log("One promise done");
|
|
let p2 = database.insertTodayDomains(db,res).then(()=>{
|
|
let p3 = database.getYesterdayDomains(db).then((result)=>{
|
|
|
|
result.map((domain)=>{
|
|
getDomainListPromises.push(database.cleanExpired(db,domain));
|
|
});
|
|
|
|
}).catch((e)=>{
|
|
console.log(e);
|
|
});
|
|
getDomainListPromises.push(p3);
|
|
});
|
|
getDomainListPromises.push(p2);
|
|
});
|
|
getDomainListPromises.push(p);
|
|
});
|
|
|
|
Promise.all(getDomainListPromises).then(()=>{
|
|
console.log("All promises done");
|
|
database.getExpiredDomains(db).then((result)=>{
|
|
links.checkExpiredDomains(db,result).then(()=>{
|
|
db.close();
|
|
});
|
|
});
|
|
});
|
|
}
|
|
});
|
|
|
|
}).catch(reason=>{
|
|
console.log("Error : " + reason);
|
|
}); |