senaduka/old-domene-svedska
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

code fix

Browse Source
This commit is contained in:
GotPPay
2017-12-12 22:08:02 +01:00
parent 6859ba39be
commit 0b7fbc6b72
5 changed files with 154 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
crawler/crawler.js
View File

@@ -1,14 +1,15 @@
var config = require('./config');
const links = require('./helper/links');
const database = require('./helper/database');
var MongoClient = require ('mongodb').MongoClient;
var ObjectID = require ('mongodb').ObjectID;
var fs = require('fs');
var datetime = require('node-datetime');
var http = require('http');
const punycode = require('punycode');
MongoClient.connect(config.databaseURL).then(database => {
let db = database;
MongoClient.connect(config.databaseURL).then(mongoDatabase => {
let db = mongoDatabase;
db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});
@@ -30,55 +31,30 @@ MongoClient.connect(config.databaseURL).then(database => {
config.domainList.map(url=>{
//get domain list from url
links.getDomainList(url, (res)=>{
db.collection('today').insert(res,()=>{
//insertion done, compare domains with yesterday
db.collection('yesterday').find({}).toArray((err,result)=>{
if (err){
console.log("Error : " + err);
}else{
result.map((domain)=>{
db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
if (result===null){
if (datetime.create().format('Y-m-d')===domain.expirationDate){
db.collection('expired_list').insert(domain);
}
}
});
links.getDomainList(url).then(res=>{
database.insertTodayDomains(db,res).then(()=>{
database.getYesterdayDomains(db).then((result)=>{
result.map((domain)=>{
db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
db.collection('expired_list').insert(domain);
}
});
db.collection('expired_list').find({}).toArray((err,result)=>{
result.map(domain=>{
let checkLink = '';
switch(domain.tld){
case 'se':
checkLink = config.seDomainCheck;
break;
case 'nu':
checkLink = config.nuDomainCheck;
break;
}
let fullName = domain.domainName + '.' + domain.tld;
http.get(checkLink+punycode.toASCII(fullName), res => {
res.setEncoding("utf8");
let body = "";
res.on("data", data => {
body += data;
});
res.on("end", () => {
let status = body.split(' ')[0];
if (status !== 'free'){
db.collection('expired_list').remove({domainName:domain.domainName});
}
});
});
});
process.exit(0);
});
database.getExpiredDomains(db).then((result)=>{
links.checkExpiredDomains(db,result).then(()=>{
console.log("Done");
db.close();
});
}
});
}).catch((e)=>{
console.log(e);
});
});
});
});
});
}
});