.

.gitignore

@@ -0,0 +1 @@
+node_modules

README.md

@@ -1,2 +1,7 @@
 # domene-svedska
 Elman domene svedska projekat
+
+
+cd crawler
+npm install
+node crawler.js

backend/config.js

@@ -0,0 +1,5 @@
+var config = {};
+
+config.PORT = 3000;
+
+module.exports = config;

backend/server.js

@@ -0,0 +1,6 @@
+const express = require('express');
+const config = require('./config');
+
+const app = express();
+
+app.listen(config.PORT, ()=>{console.log('Server running on port ' + config.PORT)});

crawler/config.js

@@ -0,0 +1,17 @@
+var config = {};
+
+config.domainList = [
+    'https://www.iis.se/data/bardate_domains_nu.txt',
+    'https://www.iis.se/data/bardate_domains.txt'];
+
+config.wordList = __dirname + '/words.txt';
+
+config.lettersOnlyRegex = /^[A-Za-z]+$/;
+
+config.databaseURL = 'mongodb://localhost:27017/kivi';
+
+
+//====
+config.words = [];
+
+module.exports = config;

crawler/crawler.js

@@ -0,0 +1,31 @@
+var config = require('./config');
+const links = require('./helper/links');
+var MongoClient = require ('mongodb').MongoClient;
+var ObjectID = require ('mongodb').ObjectID;
+var fs = require('fs');
+
+MongoClient.connect(config.databaseURL).then(database => {
+    let db = database;
+    //db.collection ('yesterday').createIndex ({domainName: 'text'});
+
+    //Get word list into memory
+    fs.readFile(config.wordList, 'utf8', (err,data)=>{
+        if (err){
+            console.log(err);
+            config.words=[];
+        }else{
+            config.words = data.split('\n');   
+        }
+    });
+
+    config.domainList.map(url=>{
+        //get domain list from url
+        links.getDomainList(url, (res)=>{
+            //res.map(obj =>{fs.appendFile('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})});
+        });
+    });
+    
+    
+}).catch(reason=>{
+    console.log("Error : " + reason);
+});

crawler/helper/links.js

@@ -0,0 +1,47 @@
+const config = require('../config');
+const https = require("https");
+
+module.exports = {
+    getDomainList : function(url, callback){
+        getRawDomainList(url,(raw)=>{
+            let result = [];
+            raw.split('\n').map(domain=>{
+                let dot = domain.indexOf('.');
+                if (dot !== -1){
+                    let domainName = domain.substring(0,dot);
+                    if (domainName.match(config.lettersOnlyRegex)){
+                        //domain name contains only letters
+                        //line in domain list is formatted as follows : [domain name]\t[expiration date]
+                        result.push({domainName: domainName, expirationDate: domain.split('\t')[1]});
+                    }
+                }
+            });
+            applyFilter(result, callback);
+        });
+    }
+};
+
+var applyFilter = function (domains, callback){
+    //get domain names that only match whole words
+    let result = [];
+    domains.map(domain=>{
+        let index = config.words.indexOf(domain.domainName);
+        if (index !== -1){
+            result.push(domain);
+        }
+    });
+    callback(result);
+}
+
+var getRawDomainList = function (url, callback) {
+    https.get(url, res => {
+        res.setEncoding("utf8");
+        let body = "";
+        res.on("data", data => {
+            body += data;
+        });
+        res.on("end", () => {
+            callback(body);
+        });
+    });
+}

crawler/package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "mongodb": "^2.2.33"
+  }
+}