full filter

crawler/config.js

@@ -1,12 +1,18 @@
 var config = {};
 
+/*
 config.domainList = [
     'https://www.iis.se/data/bardate_domains_nu.txt',
     'https://www.iis.se/data/bardate_domains.txt'];
 
+*/
+
+config.domainList = ['/home/bilal/Saburly/domene-svedska/crawler/bardate_domains.txt'];
+
 config.wordList = __dirname + '/words.txt';
 
 config.lettersOnlyRegex = /^[A-Za-z]+$/;
+config.swedishLettersOnly = /^[A-Za-zÅåÄäÖöüÜáÁèÈàÀéÉëËíÍÆæøØçÇ]+$/;
 
 config.databaseURL = 'mongodb://localhost:27017/kivi';
 

crawler/crawler.js

@@ -15,16 +15,21 @@ MongoClient.connect(config.databaseURL).then(database => {
             config.words=[];
         }else{
             config.words = data.split('\n');
-        }
+            let tmpWords = config.words.map((word,index)=>{
+                return word.toLowerCase();
             });
+            config.words = tmpWords;
 
             config.domainList.map(url=>{
                 //get domain list from url
                 links.getDomainList(url, (res)=>{
-            res.map(obj =>{fs.appendFile('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})});
+                    res.map(obj =>{
+                        fs.appendFileSync('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})
                     });
                 });
-    
+            });
+        }
+    });
     
 }).catch(reason=>{
     console.log("Error : " + reason);

crawler/helper/links.js

@@ -1,15 +1,19 @@
 const config = require('../config');
 const https = require("https");
+const punycode = require('punycode');
+var fs = require('fs');
 
 module.exports = {
     getDomainList : function(url, callback){
         getRawDomainList(url,(raw)=>{
+
             let result = [];
             raw.split('\n').map(domain=>{
-                let dot = domain.indexOf('.');
+                let unicodeDomain = punycode.toUnicode(domain);
+                let dot = unicodeDomain.indexOf('.');
                 if (dot !== -1){
-                    let domainName = domain.substring(0,dot);
+                    let domainName = unicodeDomain.substring(0,dot);
-                    if (domainName.match(config.lettersOnlyRegex)){
+                    if (domainName.match(config.swedishLettersOnly)){
                         //domain name contains only letters
                         //line in domain list is formatted as follows : [domain name]\t[expiration date]
                         result.push({domainName: domainName, expirationDate: domain.split('\t')[1]});
@@ -17,6 +21,7 @@ module.exports = {
                 }
             });
             applyFilter(result, callback);
+            console.log("Result Len : " + result.length);
         });
     }
 };
@@ -34,6 +39,16 @@ var applyFilter = function (domains, callback){
 }
 
 var getRawDomainList = function (url, callback) {
+    if (url[0]==='/'){
+        //it's local file
+        fs.readFile(url,'utf8',(err,data)=>{
+            if (err){
+                console.log("err : " + err);
+            }else{
+                callback(data);
+            }
+        });
+    }else{
         https.get(url, res => {
             res.setEncoding("utf8");
             let body = "";
@@ -45,3 +60,4 @@ var getRawDomainList = function (url, callback) {
             });
         });
     }
+}

crawler/package.json

@@ -1,5 +1,6 @@
 {
   "dependencies": {
-    "mongodb": "^2.2.33"
+    "mongodb": "^2.2.33",
+    "punycode": "^2.1.0"
   }
 }