filter with date ; working backend and frontend

crawler/bardate_domains.txt

@@ -3260,7 +3260,7 @@ araf.se	2018-01-15
 aramesk.se	2018-01-31
 aranetwork.se	2018-01-27
 aranyani.se	2018-01-10
-arap.se	2018-01-15
+arap.se	2017-12-15
 arastor.se	2018-01-26
 aratoun.se	2017-12-26
 aravia.se	2018-02-10

crawler/config.js

@@ -7,7 +7,10 @@ config.domainList = [
 
 
 
-//config.domainList = ['/home/bilal/Saburly/domene-svedska/crawler/bardate_domains.txt'];
+//config.domainList = ['/home/bilal/Saburly/domene-svedska/crawler/bardate_domains_11_12.txt'];
+
+config.seDomainCheck = 'http://free.iis.se/free?q=';
+config.nuDomainCheck = 'http://free.iis.nu/free?q=';
 
 config.wordList = __dirname + '/words.txt';
 

crawler/crawler.js

@@ -3,10 +3,19 @@ const links = require('./helper/links');
 var MongoClient = require ('mongodb').MongoClient;
 var ObjectID = require ('mongodb').ObjectID;
 var fs = require('fs');
+var datetime = require('node-datetime');
+var http = require('http');
+const punycode = require('punycode');
 
 MongoClient.connect(config.databaseURL).then(database => {
     let db = database;
-    //db.collection ('yesterday').createIndex ({domainName: 'text'});
+
+    db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
+    db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});
+
+    db.collection('yesterday').drop();
+    db.collection('today').rename('yesterday');
+    db.createCollection('today');
 
     //Get word list into memory
     fs.readFile(config.wordList, 'utf8', (err,data)=>{
@@ -15,22 +24,65 @@ MongoClient.connect(config.databaseURL).then(database => {
             config.words=[];
         }else{
             config.words = data.split('\n');
-            let tmpWords = config.words.map((word,index)=>{
+            config.words = config.words.map(word=>{
                 return word.toLowerCase();
             });
-            config.words = tmpWords;
 
+            
             config.domainList.map(url=>{
                 //get domain list from url
                 links.getDomainList(url, (res)=>{
-                    res.map(obj =>{
-                        fs.appendFileSync('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})
+                    db.collection('today').insert(res,()=>{
+                        //insertion done, compare domains with yesterday
+                        db.collection('yesterday').find({}).toArray((err,result)=>{
+                            if (err){
+                                console.log("Error : " + err);
+                            }else{
+                                result.map((domain)=>{
+                                    db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
+                                        if (result===null){
+                                            if (datetime.create().format('Y-m-d')===domain.expirationDate){
+                                                db.collection('expired_list').insert(domain);
+                                            }
+                                        }
+                                    });
+                                });
+                                db.collection('expired_list').find({}).toArray((err,result)=>{
+                                    result.map(domain=>{
+                                        let checkLink = '';
+                                        switch(domain.tld){
+                                            case 'se':
+                                                checkLink = config.seDomainCheck;
+                                                break;
+                                            case 'nu':
+                                                checkLink = config.nuDomainCheck;
+                                                break;
+                                        }
+                                        let fullName = domain.domainName + '.' + domain.tld;
+                                        http.get(checkLink+punycode.toASCII(fullName), res => {
+                                            res.setEncoding("utf8");
+                                            let body = "";
+                                            res.on("data", data => {
+                                                body += data;
+                                            });
+                                            res.on("end", () => {
+                                                let status = body.split(' ')[0];
+                                                if (status !== 'free'){
+                                                    db.collection('expired_list').remove({domainName:domain.domainName});
+                                                }
+                                            });
+                                        });
+                                    });
+                                    process.exit(0);
+                                });
+                            }
+                        });
                     });
                 });
             });
         }
     });
-    
+
 }).catch(reason=>{
     console.log("Error : " + reason);
 });

crawler/helper/links.js

@@ -5,18 +5,20 @@ var fs = require('fs');
 
 module.exports = {
     getDomainList : function(url, callback){
+        
         getRawDomainList(url,(raw)=>{
-
             let result = [];
             raw.split('\n').map(domain=>{
                 let unicodeDomain = punycode.toUnicode(domain);
                 let dot = unicodeDomain.indexOf('.');
+                let tab = unicodeDomain.indexOf('\t');
                 if (dot !== -1){
                     let domainName = unicodeDomain.substring(0,dot);
+                    let tld = unicodeDomain.substring(dot+1,tab);
                     if (domainName.match(config.swedishLettersOnly)){
                         //domain name contains only letters
                         //line in domain list is formatted as follows : [domain name]\t[expiration date]
-                        result.push({domainName: domainName, expirationDate: domain.split('\t')[1]});
+                        result.push({domainName: domainName, tld:tld ,expirationDate: domain.split('\t')[1]});
                     }
                 }
             });

crawler/package.json

@@ -1,6 +1,7 @@
 {
   "dependencies": {
     "mongodb": "^2.2.33",
+    "node-datetime": "^2.0.3",
     "punycode": "^2.1.0"
   }
 }