old-domene-svedska/crawler/crawler.js

var config = require('./config');
const links = require('./helper/links');
const database = require('./helper/database');
var MongoClient = require ('mongodb').MongoClient;
var ObjectID = require ('mongodb').ObjectID;
var fs = require('fs');
var datetime = require('node-datetime');
var http = require('http');


MongoClient.connect(config.databaseURL).then(mongoDatabase => {
    let db = mongoDatabase;

    db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
    db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});

    db.collection('yesterday').drop();
    db.collection('today').rename('yesterday');
    db.createCollection('today');

    //Get word list into memory
    fs.readFile(config.wordList, 'utf8', (err,data)=>{
        if (err){
            console.log(err);
            config.words=[];
        }else{
            config.words = data.split('\n');
            config.words = config.words.map(word=>{
                return word.toLowerCase();
            });

            
            config.domainList.map(url=>{
                links.getDomainList(url).then(res=>{
                    database.insertTodayDomains(db,res).then(()=>{
                        database.getYesterdayDomains(db).then((result)=>{

                            result.map((domain)=>{
                                db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
                                    if ((result===null) &&(datetime.create().format('Y-m-d')===domain.expirationDate)){
                                        db.collection('expired_list').insert(domain);
                                    }
                                });
                            });

                            database.getExpiredDomains(db).then((result)=>{
                                links.checkExpiredDomains(db,result).then(()=>{
                                    console.log("Done");
                                    db.close();
                                });
                            });

                        }).catch((e)=>{
                            console.log(e);
                        });
                    });
                });                
            });
        }
    });

}).catch(reason=>{
    console.log("Error : " + reason);
});