senaduka/old-domene-svedska
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

code fix

Browse Source
This commit is contained in:
GotPPay
2017-12-12 22:08:02 +01:00
parent 6859ba39be
commit 0b7fbc6b72
5 changed files with 154 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/bardate_domains_10_12.txt
View File

@@ -1248,7 +1248,7 @@ agag.se 2018-01-15
agagil.se 2018-02-13 agagil.se 2018-02-13
agal.se 2018-01-15 agal.se 2018-01-15
agaler.se 2018-02-13 agaler.se 2018-02-13
agan.se 2018-01-15 agan.se 2017-12-12
agapolitikern.se 2018-01-08 agapolitikern.se 2018-01-08
agarum.se 2017-12-30 agarum.se 2017-12-30
agathared.se 2017-12-29 agathared.se 2017-12-29

1
crawler/bardate_domains_11_12.txt
View File

@@ -1252,7 +1252,6 @@ agag.se 2018-01-15
agagil.se 2018-02-13 agagil.se 2018-02-13
agal.se 2018-01-15 agal.se 2018-01-15
agaler.se 2018-02-13 agaler.se 2018-02-13
agan.se 2018-01-15
agapolitikern.se 2018-01-08 agapolitikern.se 2018-01-08
agarum.se 2017-12-30 agarum.se 2017-12-30
agathared.se 2017-12-29 agathared.se 2017-12-29

72
crawler/crawler.js
View File

@@ -1,14 +1,15 @@
var config = require('./config'); var config = require('./config');
const links = require('./helper/links'); const links = require('./helper/links');
const database = require('./helper/database');
var MongoClient = require ('mongodb').MongoClient; var MongoClient = require ('mongodb').MongoClient;
var ObjectID = require ('mongodb').ObjectID; var ObjectID = require ('mongodb').ObjectID;
var fs = require('fs'); var fs = require('fs');
var datetime = require('node-datetime'); var datetime = require('node-datetime');
var http = require('http'); var http = require('http');
const punycode = require('punycode');
MongoClient.connect(config.databaseURL).then(database => {
let db = database; MongoClient.connect(config.databaseURL).then(mongoDatabase => {
let db = mongoDatabase;
db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true}); db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true}); db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});
@@ -30,55 +31,30 @@ MongoClient.connect(config.databaseURL).then(database => {
config.domainList.map(url=>{ config.domainList.map(url=>{
//get domain list from url links.getDomainList(url).then(res=>{
links.getDomainList(url, (res)=>{ database.insertTodayDomains(db,res).then(()=>{
db.collection('today').insert(res,()=>{ database.getYesterdayDomains(db).then((result)=>{
//insertion done, compare domains with yesterday
db.collection('yesterday').find({}).toArray((err,result)=>{ result.map((domain)=>{
if (err){ db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
console.log("Error : " + err); if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
}else{ db.collection('expired_list').insert(domain);
result.map((domain)=>{ }
db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
if (result===null){
if (datetime.create().format('Y-m-d')===domain.expirationDate){
db.collection('expired_list').insert(domain);
}
}
});
}); });
db.collection('expired_list').find({}).toArray((err,result)=>{ });
result.map(domain=>{
let checkLink = ''; database.getExpiredDomains(db).then((result)=>{
switch(domain.tld){ links.checkExpiredDomains(db,result).then(()=>{
case 'se': console.log("Done");
checkLink = config.seDomainCheck; db.close();
break;
case 'nu':
checkLink = config.nuDomainCheck;
break;
}
let fullName = domain.domainName + '.' + domain.tld;
http.get(checkLink+punycode.toASCII(fullName), res => {
res.setEncoding("utf8");
let body = "";
res.on("data", data => {
body += data;
});
res.on("end", () => {
let status = body.split(' ')[0];
if (status !== 'free'){
db.collection('expired_list').remove({domainName:domain.domainName});
}
});
});
});
process.exit(0);
}); });
} });
}).catch((e)=>{
console.log(e);
}); });
}); });
}); });
}); });
} }
}); });

31
crawler/helper/database.js Normal file
View File

@@ -0,0 +1,31 @@
module.exports = {
insertTodayDomains : function (db, domains){
return new Promise((resolve,reject)=>{
db.collection('today').insert(domains, ()=>{
resolve();
});
});
},
getYesterdayDomains : function (db){
return getArrayFromDatabase(db, 'yesterday');
},
getExpiredDomains : function (db){
return getArrayFromDatabase(db, 'expired_list');
}
};
const getArrayFromDatabase = function(db, collection){
return new Promise((resolve,reject)=>{
db.collection(collection).find({}).toArray((err,result)=>{
if (err){
reject('Error reading collection');
}else{
resolve(result);
}
});
});
}

147
crawler/helper/links.js
View File

@@ -4,61 +4,110 @@ const punycode = require('punycode');
var fs = require('fs'); var fs = require('fs');
module.exports = { module.exports = {
getDomainList : function(url, callback){ getDomainList : function(url){
return new Promise((resolve, reject)=>{
getRawDomainList(url,(raw)=>{ getRawDomainList(url).then(raw=>{
let result = []; processDomains(raw).then(result=>{
raw.split('\n').map(domain=>{ applyFilter(result).then(result=>{
let unicodeDomain = punycode.toUnicode(domain); resolve(result);
let dot = unicodeDomain.indexOf('.'); })
let tab = unicodeDomain.indexOf('\t'); });
if (dot !== -1){
let domainName = unicodeDomain.substring(0,dot);
let tld = unicodeDomain.substring(dot+1,tab);
if (domainName.match(config.swedishLettersOnly)){
//domain name contains only letters
//line in domain list is formatted as follows : [domain name]\t[expiration date]
result.push({domainName: domainName, tld:tld ,expirationDate: domain.split('\t')[1]});
}
}
}); });
applyFilter(result, callback); });
},
checkExpiredDomains : function(db, domains){
return new Promise((resolve,reject)=>{
domains.map(domain=>{
let checkLink = '';
switch(domain.tld){
case 'se':
checkLink = config.seDomainCheck;
break;
case 'nu':
checkLink = config.nuDomainCheck;
break;
}
let fullName = domain.domainName + '.' + domain.tld;
http.get(checkLink+punycode.toASCII(fullName), res => {
res.setEncoding("utf8");
let body = "";
res.on("data", data => {
body += data;
});
res.on("end", () => {
let status = body.split(' ')[0];
if (status !== 'free'){
db.collection('expired_list').remove({domainName:domain.domainName});
}
});
});
});
resolve();
}); });
} }
}; };
var applyFilter = function (domains, callback){ var applyFilter = function (domains){
//get domain names that only match whole words return new Promise((resolve,reject)=>{
let result = []; //get domain names that only match whole words
domains.map(domain=>{ let result = [];
let index = config.words.indexOf(domain.domainName); domains.map(domain=>{
if (index !== -1){ let index = config.words.indexOf(domain.domainName);
result.push(domain); if (index !== -1){
} result.push(domain);
});
callback(result);
}
var getRawDomainList = function (url, callback) {
if (url[0]==='/'){
//it's local file
fs.readFile(url,'utf8',(err,data)=>{
if (err){
console.log("err : " + err);
}else{
callback(data);
} }
}); });
}else{ resolve(result);
https.get(url, res => { });
res.setEncoding("utf8"); }
let body = "";
res.on("data", data => { var processDomains = function(raw){
body += data; return new Promise((resolve,reject)=>{
}); let result = [];
res.on("end", () => { raw.split('\n').map(domain=>{
callback(body); let unicodeDomain = punycode.toUnicode(domain);
}); let dot = unicodeDomain.indexOf('.');
let tab = unicodeDomain.indexOf('\t');
if (dot !== -1){
let domainName = unicodeDomain.substring(0,dot);
let tld = unicodeDomain.substring(dot+1,tab);
if (domainName.match(config.swedishLettersOnly)){
//domain name contains only letters
//line in domain list is formatted as follows : [domain name]\t[expiration date]
result.push({domainName: domainName, tld:tld ,expirationDate: domain.split('\t')[1]});
}
}
}); });
} resolve(result);
});
}
var getRawDomainList = function (url) {
return new Promise((resolve, reject)=>{
if (url[0]==='/'){
//it's local file
fs.readFile(url,'utf8',(err,data)=>{
if (err){
reject(err);
}else{
resolve(data);
}
});
}else{
https.get(url, res => {
res.setEncoding("utf8");
let body = "";
res.on("data", data => {
body += data;
});
res.on("end", () => {
resolve(body);
});
});
}
});
} }