senaduka/old-domene-svedska
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

full filter

Browse Source
This commit is contained in:
GotPPay
2017-12-08 10:57:47 +01:00
parent 100299b327
commit c69b45b632
8 changed files with 184323 additions and 78507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
crawler/crawler.js
View File

@@ -14,17 +14,22 @@ MongoClient.connect(config.databaseURL).then(database => {
console.log(err);
config.words=[];
}else{
config.words = data.split('\n');
config.words = data.split('\n');
let tmpWords = config.words.map((word,index)=>{
return word.toLowerCase();
});
config.words = tmpWords;
config.domainList.map(url=>{
//get domain list from url
links.getDomainList(url, (res)=>{
res.map(obj =>{
fs.appendFileSync('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})
});
});
});
}
});
config.domainList.map(url=>{
//get domain list from url
links.getDomainList(url, (res)=>{
res.map(obj =>{fs.appendFile('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})});
});
});
}).catch(reason=>{
console.log("Error : " + reason);