code fix

crawler/bardate_domains_10_12.txt

@@ -1248,7 +1248,7 @@ agag.se	2018-01-15
 agagil.se	2018-02-13
 agal.se	2018-01-15
 agaler.se	2018-02-13
-agan.se	2018-01-15
+agan.se	2017-12-12
 agapolitikern.se	2018-01-08
 agarum.se	2017-12-30
 agathared.se	2017-12-29

crawler/bardate_domains_11_12.txt

@@ -1252,7 +1252,6 @@ agag.se	2018-01-15
 agagil.se	2018-02-13
 agal.se	2018-01-15
 agaler.se	2018-02-13
-agan.se	2018-01-15
 agapolitikern.se	2018-01-08
 agarum.se	2017-12-30
 agathared.se	2017-12-29

crawler/crawler.js

@@ -1,14 +1,15 @@
 var config = require('./config');
 const links = require('./helper/links');
+const database = require('./helper/database');
 var MongoClient = require ('mongodb').MongoClient;
 var ObjectID = require ('mongodb').ObjectID;
 var fs = require('fs');
 var datetime = require('node-datetime');
 var http = require('http');
-const punycode = require('punycode');
 
-MongoClient.connect(config.databaseURL).then(database => {
+
-    let db = database;
+MongoClient.connect(config.databaseURL).then(mongoDatabase => {
+    let db = mongoDatabase;
 
     db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
     db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});
@@ -30,52 +31,27 @@ MongoClient.connect(config.databaseURL).then(database => {
 
             
             config.domainList.map(url=>{
-                //get domain list from url
+                links.getDomainList(url).then(res=>{
-                links.getDomainList(url, (res)=>{
+                    database.insertTodayDomains(db,res).then(()=>{
-                    db.collection('today').insert(res,()=>{
+                        database.getYesterdayDomains(db).then((result)=>{
-                        //insertion done, compare domains with yesterday
+
-                        db.collection('yesterday').find({}).toArray((err,result)=>{
-                            if (err){
-                                console.log("Error : " + err);
-                            }else{
                             result.map((domain)=>{
                                 db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
-                                        if (result===null){
+                                    if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
-                                            if (datetime.create().format('Y-m-d')===domain.expirationDate){
                                         db.collection('expired_list').insert(domain);
                                     }
-                                        }
                                 });
                             });
-                                db.collection('expired_list').find({}).toArray((err,result)=>{
+
-                                    result.map(domain=>{
+                            database.getExpiredDomains(db).then((result)=>{
-                                        let checkLink = '';
+                                links.checkExpiredDomains(db,result).then(()=>{
-                                        switch(domain.tld){
+                                    console.log("Done");
-                                            case 'se':
+                                    db.close();
-                                                checkLink = config.seDomainCheck;
-                                                break;
-                                            case 'nu':
-                                                checkLink = config.nuDomainCheck;
-                                                break;
-                                        }
-                                        let fullName = domain.domainName + '.' + domain.tld;
-                                        http.get(checkLink+punycode.toASCII(fullName), res => {
-                                            res.setEncoding("utf8");
-                                            let body = "";
-                                            res.on("data", data => {
-                                                body += data;
-                                            });
-                                            res.on("end", () => {
-                                                let status = body.split(' ')[0];
-                                                if (status !== 'free'){
-                                                    db.collection('expired_list').remove({domainName:domain.domainName});
-                                                }
                                 });
                             });
-                                    });
+
-                                    process.exit(0);
+                        }).catch((e)=>{
-                                });
+                            console.log(e);
-                            }
                         });
                     });
                 });                

crawler/helper/database.js

@@ -0,0 +1,31 @@
+module.exports = {
+    insertTodayDomains : function (db, domains){
+        return new Promise((resolve,reject)=>{
+            db.collection('today').insert(domains, ()=>{
+                resolve();
+            });
+        });
+    },
+
+    getYesterdayDomains : function (db){
+        return getArrayFromDatabase(db, 'yesterday');
+    },
+
+    getExpiredDomains : function (db){
+        return getArrayFromDatabase(db, 'expired_list');
+    }
+
+
+};
+
+const getArrayFromDatabase = function(db, collection){
+    return new Promise((resolve,reject)=>{
+        db.collection(collection).find({}).toArray((err,result)=>{
+            if (err){
+                reject('Error reading collection');
+            }else{
+                resolve(result);
+            }
+        });
+    });
+}

crawler/helper/links.js

@@ -4,9 +4,69 @@ const punycode = require('punycode');
 var fs = require('fs');
 
 module.exports = {
-    getDomainList : function(url, callback){
+    getDomainList : function(url){
+        return new Promise((resolve, reject)=>{
+            getRawDomainList(url).then(raw=>{
+                processDomains(raw).then(result=>{
+                    applyFilter(result).then(result=>{
+                        resolve(result);
+                    })
+                });
+            });
+        });
+    },
 
-        getRawDomainList(url,(raw)=>{
+    checkExpiredDomains : function(db, domains){
+        return new Promise((resolve,reject)=>{
+            domains.map(domain=>{
+                let checkLink = '';
+                switch(domain.tld){
+                    case 'se':
+                        checkLink = config.seDomainCheck;
+                        break;
+                    case 'nu':
+                        checkLink = config.nuDomainCheck;
+                        break;
+                }
+    
+                let fullName = domain.domainName + '.' + domain.tld;
+                http.get(checkLink+punycode.toASCII(fullName), res => {
+                    res.setEncoding("utf8");
+                    let body = "";
+                    res.on("data", data => {
+                        body += data;
+                    });
+                    res.on("end", () => {
+                        let status = body.split(' ')[0];
+                        if (status !== 'free'){
+                            db.collection('expired_list').remove({domainName:domain.domainName});
+                        }
+                    });
+                });
+            });
+            resolve();
+        });
+    }
+
+    
+};
+
+var applyFilter = function (domains){
+    return new Promise((resolve,reject)=>{
+        //get domain names that only match whole words
+        let result = [];
+        domains.map(domain=>{
+            let index = config.words.indexOf(domain.domainName);
+            if (index !== -1){
+                result.push(domain);
+            }
+        });
+        resolve(result);
+    });
+}
+
+var processDomains = function(raw){
+    return new Promise((resolve,reject)=>{
         let result = [];
         raw.split('\n').map(domain=>{
             let unicodeDomain = punycode.toUnicode(domain);
@@ -22,31 +82,19 @@ module.exports = {
                 }
             }
         });
-            applyFilter(result, callback);
+        resolve(result);
     });
-    }
-};
-
-var applyFilter = function (domains, callback){
-    //get domain names that only match whole words
-    let result = [];
-    domains.map(domain=>{
-        let index = config.words.indexOf(domain.domainName);
-        if (index !== -1){
-            result.push(domain);
-        }
-    });
-    callback(result);
 }
 
-var getRawDomainList = function (url, callback) {
+var getRawDomainList = function (url) {
+    return new Promise((resolve, reject)=>{
         if (url[0]==='/'){
             //it's local file
             fs.readFile(url,'utf8',(err,data)=>{
                 if (err){
-                console.log("err : " + err);
+                    reject(err);
                 }else{
-                callback(data);
+                    resolve(data);
                 }
             });
         }else{
@@ -57,8 +105,9 @@ var getRawDomainList = function (url, callback) {
                     body += data;
                 });
                 res.on("end", () => {
-                callback(body);
+                    resolve(body);
                 });
             });
         }
+    });
 }