senaduka/old-domene-svedska
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filter with date ; working backend and frontend

Browse Source
This commit is contained in:
GotPPay
2017-12-11 18:22:06 +01:00
parent b634bfd300
commit 6859ba39be
26 changed files with 128932 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
crawler/crawler.js
View File

@@ -3,10 +3,19 @@ const links = require('./helper/links');
var MongoClient = require ('mongodb').MongoClient;
var ObjectID = require ('mongodb').ObjectID;
var fs = require('fs');
var datetime = require('node-datetime');
var http = require('http');
const punycode = require('punycode');
MongoClient.connect(config.databaseURL).then(database => {
let db = database;
//db.collection ('yesterday').createIndex ({domainName: 'text'});
db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});
db.collection('yesterday').drop();
db.collection('today').rename('yesterday');
db.createCollection('today');
//Get word list into memory
fs.readFile(config.wordList, 'utf8', (err,data)=>{
@@ -15,22 +24,65 @@ MongoClient.connect(config.databaseURL).then(database => {
config.words=[];
}else{
config.words = data.split('\n');
let tmpWords = config.words.map((word,index)=>{
config.words = config.words.map(word=>{
return word.toLowerCase();
});
config.words = tmpWords;
config.domainList.map(url=>{
//get domain list from url
links.getDomainList(url, (res)=>{
res.map(obj =>{
fs.appendFileSync('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})
db.collection('today').insert(res,()=>{
//insertion done, compare domains with yesterday
db.collection('yesterday').find({}).toArray((err,result)=>{
if (err){
console.log("Error : " + err);
}else{
result.map((domain)=>{
db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
if (result===null){
if (datetime.create().format('Y-m-d')===domain.expirationDate){
db.collection('expired_list').insert(domain);
}
}
});
});
db.collection('expired_list').find({}).toArray((err,result)=>{
result.map(domain=>{
let checkLink = '';
switch(domain.tld){
case 'se':
checkLink = config.seDomainCheck;
break;
case 'nu':
checkLink = config.nuDomainCheck;
break;
}
let fullName = domain.domainName + '.' + domain.tld;
http.get(checkLink+punycode.toASCII(fullName), res => {
res.setEncoding("utf8");
let body = "";
res.on("data", data => {
body += data;
});
res.on("end", () => {
let status = body.split(' ')[0];
if (status !== 'free'){
db.collection('expired_list').remove({domainName:domain.domainName});
}
});
});
});
process.exit(0);
});
}
});
});
});
});
}
});
}).catch(reason=>{
console.log("Error : " + reason);
});