.gitignore

@@ -0,0 +1 @@
+node_modules

README.md

@@ -1,2 +1,7 @@
 # domene-svedska
 Elman domene svedska projekat
+
+
+cd crawler
+npm install
+node crawler.js

backend/config.js

@@ -0,0 +1,5 @@
+var config = {};
+
+config.PORT = 3000;
+
+module.exports = config;

backend/server.js

@@ -0,0 +1,6 @@
+const express = require('express');
+const config = require('./config');
+
+const app = express();
+
+app.listen(config.PORT, ()=>{console.log('Server running on port ' + config.PORT)});

crawler/config.js

@@ -0,0 +1,23 @@
+var config = {};
+
+
+config.domainList = [
+    'https://www.iis.se/data/bardate_domains_nu.txt',
+    'https://www.iis.se/data/bardate_domains.txt'];
+
+
+
+//config.domainList = ['/home/bilal/Saburly/domene-svedska/crawler/bardate_domains.txt'];
+
+config.wordList = __dirname + '/words.txt';
+
+config.lettersOnlyRegex = /^[A-Za-z]+$/;
+config.swedishLettersOnly = /^[A-Za-zÅåÄäÖöüÜáÁèÈàÀéÉëËíÍÆæøØçÇ]+$/;
+
+config.databaseURL = 'mongodb://localhost:27017/domains';
+
+
+//====
+config.words = [];
+
+module.exports = config;

crawler/crawler.js

@@ -0,0 +1,36 @@
+var config = require('./config');
+const links = require('./helper/links');
+var MongoClient = require ('mongodb').MongoClient;
+var ObjectID = require ('mongodb').ObjectID;
+var fs = require('fs');
+
+MongoClient.connect(config.databaseURL).then(database => {
+    let db = database;
+    //db.collection ('yesterday').createIndex ({domainName: 'text'});
+
+    //Get word list into memory
+    fs.readFile(config.wordList, 'utf8', (err,data)=>{
+        if (err){
+            console.log(err);
+            config.words=[];
+        }else{
+            config.words = data.split('\n');
+            let tmpWords = config.words.map((word,index)=>{
+                return word.toLowerCase();
+            });
+            config.words = tmpWords;
+
+            config.domainList.map(url=>{
+                //get domain list from url
+                links.getDomainList(url, (res)=>{
+                    res.map(obj =>{
+                        fs.appendFileSync('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})
+                    });
+                });
+            });
+        }
+    });
+    
+}).catch(reason=>{
+    console.log("Error : " + reason);
+});

crawler/helper/links.js

@@ -0,0 +1,62 @@
+const config = require('../config');
+const https = require("https");
+const punycode = require('punycode');
+var fs = require('fs');
+
+module.exports = {
+    getDomainList : function(url, callback){
+        getRawDomainList(url,(raw)=>{
+
+            let result = [];
+            raw.split('\n').map(domain=>{
+                let unicodeDomain = punycode.toUnicode(domain);
+                let dot = unicodeDomain.indexOf('.');
+                if (dot !== -1){
+                    let domainName = unicodeDomain.substring(0,dot);
+                    if (domainName.match(config.swedishLettersOnly)){
+                        //domain name contains only letters
+                        //line in domain list is formatted as follows : [domain name]\t[expiration date]
+                        result.push({domainName: domainName, expirationDate: domain.split('\t')[1]});
+                    }
+                }
+            });
+            applyFilter(result, callback);
+        });
+    }
+};
+
+var applyFilter = function (domains, callback){
+    //get domain names that only match whole words
+    let result = [];
+    domains.map(domain=>{
+        let index = config.words.indexOf(domain.domainName);
+        if (index !== -1){
+            result.push(domain);
+        }
+    });
+    callback(result);
+}
+
+var getRawDomainList = function (url, callback) {
+    if (url[0]==='/'){
+        //it's local file
+        fs.readFile(url,'utf8',(err,data)=>{
+            if (err){
+                console.log("err : " + err);
+            }else{
+                callback(data);
+            }
+        });
+    }else{
+        https.get(url, res => {
+            res.setEncoding("utf8");
+            let body = "";
+            res.on("data", data => {
+                body += data;
+            });
+            res.on("end", () => {
+                callback(body);
+            });
+        });
+    }
+}

crawler/package.json

@@ -0,0 +1,6 @@
+{
+  "dependencies": {
+    "mongodb": "^2.2.33",
+    "punycode": "^2.1.0"
+  }
+}