old-domene-svedska/crawler/helper/links.js

const config = require('../config');
const https = require("https");
const punycode = require('punycode');
var fs = require('fs');
var http = require('http');

module.exports = {
    getDomainList : function(url){
        return new Promise((resolve, reject)=>{

            getRawDomainList(url).then(raw=>{
                resolve(applyFilter(processDomains(raw)));
            });
        });
    },

    checkExpiredDomains : function(db, domains){
        return new Promise((resolve,reject)=>{
            let waitingPromises = [];
            let domainsForRemoval = [];
            domains.map(domain=>{
                let checkLink = '';
                switch(domain.tld){
                    case 'se':
                        checkLink = config.seDomainCheck;
                        break;
                    case 'nu':
                        checkLink = config.nuDomainCheck;
                        break;
                }
    
                let fullName = domain.domainName + '.' + domain.tld;
                waitingPromises.push(removeOccupiedDomain(db, checkLink+punycode.toASCII(fullName),domain));
            });
            
            Promise.all(waitingPromises).then(()=>{
                resolve();
            });
        });
    }
};

var isDomainFree = function (db, url, domain){
    return new Promise((resolve,reject)=>{
        http.get(url, res => {
            res.setEncoding("utf8");
            let body = "";
            res.on("data", data => {
                body += data;
            });
            res.on("end", () => {
                let status = body.split(' ')[0];
                if (status === 'free'){
                    resolve(true);
                }else{
                    resolve(false);
                }
            });
        });
    });
}

var removeOccupiedDomain = function (db, url,domain){
    return new Promise((resolve,reject)=>{
        http.get(url, res => {
            res.setEncoding("utf8");
            let body = "";
            res.on("data", data => {
                body += data;
            });
            res.on("end", () => {
                let status = body.split(' ')[0];
                if (status !== 'free'){
                    db.collection('expired_list').remove({domainName:domain.domainName, tld: domain.tld});
                }
                resolve();
            });
        });
    });
}

var applyFilter = function (domains){
    let result = [];
    domains.map(domain=>{
        let index = config.words.indexOf(domain.domainName);
        if (index !== -1){
            result.push(domain);
        }
    });
    return result;
}

var processDomains = function(raw){
    let result = [];
    raw.split('\n').map(domain=>{
        let unicodeDomain = punycode.toUnicode(domain);
        let dot = unicodeDomain.indexOf('.');
        let tab = unicodeDomain.indexOf('\t');
        if (dot !== -1){
            let domainName = unicodeDomain.substring(0,dot);
            let tld = unicodeDomain.substring(dot+1,tab);
            if (domainName.match(config.swedishLettersOnly)){
                //domain name contains only letters
                //line in domain list is formatted as follows : [domain name]\t[expiration date]
                result.push({domainName: domainName, tld:tld ,expirationDate: domain.split('\t')[1]});
            }
        }
    });
    return result;
}

var getRawDomainList = function (url) {
    return new Promise((resolve, reject)=>{
        if (url[0]==='/'){
            //it's local file
            fs.readFile(url,'utf8',(err,data)=>{
                if (err){
                    reject(err);
                }else{
                    resolve(data);
                }
            });
        }else{
            https.get(url, res => {
                res.setEncoding("utf8");
                let body = "";
                res.on("data", data => {
                    body += data;
                });
                res.on("end", () => {
                    resolve(body);
                });
            });
        }
    });
}