old-domene-svedska/crawler/helper/links.js

const config = require('../config');
const https = require("https");
const punycode = require('punycode');
var fs = require('fs');

module.exports = {
    getDomainList : function(url, callback){
        getRawDomainList(url,(raw)=>{

            let result = [];
            raw.split('\n').map(domain=>{
                let unicodeDomain = punycode.toUnicode(domain);
                let dot = unicodeDomain.indexOf('.');
                if (dot !== -1){
                    let domainName = unicodeDomain.substring(0,dot);
                    if (domainName.match(config.swedishLettersOnly)){
                        //domain name contains only letters
                        //line in domain list is formatted as follows : [domain name]\t[expiration date]
                        result.push({domainName: domainName, expirationDate: domain.split('\t')[1]});
                    }
                }
            });
            applyFilter(result, callback);
            console.log("Result Len : " + result.length);
        });
    }
};

var applyFilter = function (domains, callback){
    //get domain names that only match whole words
    let result = [];
    domains.map(domain=>{
        let index = config.words.indexOf(domain.domainName);
        if (index !== -1){
            result.push(domain);
        }
    });
    callback(result);
}

var getRawDomainList = function (url, callback) {
    if (url[0]==='/'){
        //it's local file
        fs.readFile(url,'utf8',(err,data)=>{
            if (err){
                console.log("err : " + err);
            }else{
                callback(data);
            }
        });
    }else{
        https.get(url, res => {
            res.setEncoding("utf8");
            let body = "";
            res.on("data", data => {
                body += data;
            });
            res.on("end", () => {
                callback(body);
            });
        });
    }
}