full filter

crawler/helper/links.js

@@ -1,15 +1,19 @@
 const config = require('../config');
 const https = require("https");
+const punycode = require('punycode');
+var fs = require('fs');
 
 module.exports = {
     getDomainList : function(url, callback){
         getRawDomainList(url,(raw)=>{
+
             let result = [];
             raw.split('\n').map(domain=>{
-                let dot = domain.indexOf('.');
+                let unicodeDomain = punycode.toUnicode(domain);
+                let dot = unicodeDomain.indexOf('.');
                 if (dot !== -1){
-                    let domainName = domain.substring(0,dot);
-                    if (domainName.match(config.lettersOnlyRegex)){
+                    let domainName = unicodeDomain.substring(0,dot);
+                    if (domainName.match(config.swedishLettersOnly)){
                         //domain name contains only letters
                         //line in domain list is formatted as follows : [domain name]\t[expiration date]
                         result.push({domainName: domainName, expirationDate: domain.split('\t')[1]});
@@ -17,6 +21,7 @@ module.exports = {
                 }
             });
             applyFilter(result, callback);
+            console.log("Result Len : " + result.length);
         });
     }
 };
@@ -34,14 +39,25 @@ var applyFilter = function (domains, callback){
 }
 
 var getRawDomainList = function (url, callback) {
-    https.get(url, res => {
-        res.setEncoding("utf8");
-        let body = "";
-        res.on("data", data => {
-            body += data;
+    if (url[0]==='/'){
+        //it's local file
+        fs.readFile(url,'utf8',(err,data)=>{
+            if (err){
+                console.log("err : " + err);
+            }else{
+                callback(data);
+            }
         });
-        res.on("end", () => {
-            callback(body);
+    }else{
+        https.get(url, res => {
+            res.setEncoding("utf8");
+            let body = "";
+            res.on("data", data => {
+                body += data;
+            });
+            res.on("end", () => {
+                callback(body);
+            });
         });
-    });
+    }
 }