old-domene-svedska/crawler/crawler.js

var config = require('./config');
const links = require('./helper/links');
const database = require('./helper/database');
var MongoClient = require ('mongodb').MongoClient;
var ObjectID = require ('mongodb').ObjectID;
var fs = require('fs');


MongoClient.connect(config.databaseURL).then(mongoDatabase => {
    let db = mongoDatabase;

    db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
    db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});

    db.collection('yesterday').drop();
    db.collection('today').rename('yesterday');
    db.createCollection('today');

    //Get word list into memory
    fs.readFile(config.wordList, 'utf8', (err,data)=>{
        if (err){
            console.log(err);
            config.words=[];
        }else{
            config.words = data.split('\n');
            config.words = config.words.map(word=>{
                return word.toLowerCase();
            });
            
            let getDomainListPromises = [];

            config.domainList.map(url=>{
                let p = links.getDomainList(url).then(res=>{
                    console.log("One promise done");
                    let p2 = database.insertTodayDomains(db,res).then(()=>{
                        let p3 = database.getYesterdayDomains(db).then((result)=>{

                            result.map((domain)=>{
                                getDomainListPromises.push(database.cleanExpired(db,domain));
                            });

                        }).catch((e)=>{
                            console.log(e);
                        });
                        getDomainListPromises.push(p3);
                    });
                    getDomainListPromises.push(p2);
                });
                getDomainListPromises.push(p);
            });

            Promise.all(getDomainListPromises).then(()=>{
                console.log("All promises done");
                database.getExpiredDomains(db).then((result)=>{
                    links.checkExpiredDomains(db,result).then(()=>{
                        db.close();
                    });
                });
            });
        }
    });

}).catch(reason=>{
    console.log("Error : " + reason);
});