filter with date ; working backend and frontend

backend/config.js

@@ -1,5 +1,7 @@
 var config = {};
 
-config.PORT = 3000;
+config.PORT = 4000;
+
+config.databaseURL = 'mongodb://localhost:27017/domains';
 
 module.exports = config;

backend/server.js

@@ -1,6 +1,23 @@
 const express = require('express');
 const config = require('./config');
 
+var MongoClient = require ('mongodb').MongoClient;
+var ObjectID = require ('mongodb').ObjectID;
+
 const app = express();
 
-app.listen(config.PORT, ()=>{console.log('Server running on port ' + config.PORT)});
+app.get('/list', (req,res)=>{
+    res.header ('Access-Control-Allow-Origin', '*');
+    db.collection('expired_list').find({}).toArray((err,result)=>{
+        if (err){
+            res.send(JSON.stringify(null));
+        }else{
+            res.send(JSON.stringify(result));
+        }
+    });
+});
+
+MongoClient.connect (config.databaseURL).then (database => {
+    db = database;
+    app.listen(config.PORT, ()=>{console.log('Server running on port ' + config.PORT)});
+});

crawler/bardate_domains.txt

@@ -3260,7 +3260,7 @@ araf.se	2018-01-15
 aramesk.se	2018-01-31
 aranetwork.se	2018-01-27
 aranyani.se	2018-01-10
-arap.se	2018-01-15
+arap.se	2017-12-15
 arastor.se	2018-01-26
 aratoun.se	2017-12-26
 aravia.se	2018-02-10

crawler/config.js

@@ -7,7 +7,10 @@ config.domainList = [
 
 
 
-//config.domainList = ['/home/bilal/Saburly/domene-svedska/crawler/bardate_domains.txt'];
+//config.domainList = ['/home/bilal/Saburly/domene-svedska/crawler/bardate_domains_11_12.txt'];
+
+config.seDomainCheck = 'http://free.iis.se/free?q=';
+config.nuDomainCheck = 'http://free.iis.nu/free?q=';
 
 config.wordList = __dirname + '/words.txt';
 

crawler/crawler.js

@@ -3,10 +3,19 @@ const links = require('./helper/links');
 var MongoClient = require ('mongodb').MongoClient;
 var ObjectID = require ('mongodb').ObjectID;
 var fs = require('fs');
+var datetime = require('node-datetime');
+var http = require('http');
+const punycode = require('punycode');
 
 MongoClient.connect(config.databaseURL).then(database => {
     let db = database;
-    //db.collection ('yesterday').createIndex ({domainName: 'text'});
+
+    db.executeDbAdminCommand( { setParameter: true, textSearchEnabled : true});
+    db.collection('expired_list').createIndex({domainName: 'text'}, {unique: true});
+
+    db.collection('yesterday').drop();
+    db.collection('today').rename('yesterday');
+    db.createCollection('today');
 
     //Get word list into memory
     fs.readFile(config.wordList, 'utf8', (err,data)=>{
@@ -15,16 +24,59 @@ MongoClient.connect(config.databaseURL).then(database => {
             config.words=[];
         }else{
             config.words = data.split('\n');
-            let tmpWords = config.words.map((word,index)=>{
+            config.words = config.words.map(word=>{
                 return word.toLowerCase();
             });
-            config.words = tmpWords;
+
             
             config.domainList.map(url=>{
                 //get domain list from url
                 links.getDomainList(url, (res)=>{
-                    res.map(obj =>{
-                        fs.appendFileSync('izlaz.txt',obj.domainName+'\n',err=>{console.log('er:' + err)})
+                    db.collection('today').insert(res,()=>{
+                        //insertion done, compare domains with yesterday
+                        db.collection('yesterday').find({}).toArray((err,result)=>{
+                            if (err){
+                                console.log("Error : " + err);
+                            }else{
+                                result.map((domain)=>{
+                                    db.collection('today').findOne({domainName:domain.domainName}, (err,result)=>{
+                                        if (result===null){
+                                            if (datetime.create().format('Y-m-d')===domain.expirationDate){
+                                                db.collection('expired_list').insert(domain);
+                                            }
+                                        }
+                                    });
+                                });
+                                db.collection('expired_list').find({}).toArray((err,result)=>{
+                                    result.map(domain=>{
+                                        let checkLink = '';
+                                        switch(domain.tld){
+                                            case 'se':
+                                                checkLink = config.seDomainCheck;
+                                                break;
+                                            case 'nu':
+                                                checkLink = config.nuDomainCheck;
+                                                break;
+                                        }
+                                        let fullName = domain.domainName + '.' + domain.tld;
+                                        http.get(checkLink+punycode.toASCII(fullName), res => {
+                                            res.setEncoding("utf8");
+                                            let body = "";
+                                            res.on("data", data => {
+                                                body += data;
+                                            });
+                                            res.on("end", () => {
+                                                let status = body.split(' ')[0];
+                                                if (status !== 'free'){
+                                                    db.collection('expired_list').remove({domainName:domain.domainName});
+                                                }
+                                            });
+                                        });
+                                    });
+                                    process.exit(0);
+                                });
+                            }
+                        });
                     });
                 });
             });

crawler/helper/links.js

@@ -5,18 +5,20 @@ var fs = require('fs');
 
 module.exports = {
     getDomainList : function(url, callback){
-        getRawDomainList(url,(raw)=>{
         
+        getRawDomainList(url,(raw)=>{
             let result = [];
             raw.split('\n').map(domain=>{
                 let unicodeDomain = punycode.toUnicode(domain);
                 let dot = unicodeDomain.indexOf('.');
+                let tab = unicodeDomain.indexOf('\t');
                 if (dot !== -1){
                     let domainName = unicodeDomain.substring(0,dot);
+                    let tld = unicodeDomain.substring(dot+1,tab);
                     if (domainName.match(config.swedishLettersOnly)){
                         //domain name contains only letters
                         //line in domain list is formatted as follows : [domain name]\t[expiration date]
-                        result.push({domainName: domainName, expirationDate: domain.split('\t')[1]});
+                        result.push({domainName: domainName, tld:tld ,expirationDate: domain.split('\t')[1]});
                     }
                 }
             });

crawler/package.json

@@ -1,6 +1,7 @@
 {
   "dependencies": {
     "mongodb": "^2.2.33",
+    "node-datetime": "^2.0.3",
     "punycode": "^2.1.0"
   }
 }

web/.gitignore

@@ -0,0 +1,21 @@
+# See https://help.github.com/ignore-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

web/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "web",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "isomorphic-fetch": "^2.2.1",
+    "react": "^16.2.0",
+    "react-dom": "^16.2.0",
+    "react-scripts": "1.0.17"
+  },
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test --env=jsdom",
+    "eject": "react-scripts eject"
+  }
+}

web/public/index.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <meta name="theme-color" content="#000000">
+    <!--
+      manifest.json provides metadata used when your web app is added to the
+      homescreen on Android. See https://developers.google.com/web/fundamentals/engage-and-retain/web-app-manifest/
+    -->
+    <link rel="manifest" href="%PUBLIC_URL%/manifest.json">
+    <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico">
+    <!--
+      Notice the use of %PUBLIC_URL% in the tags above.
+      It will be replaced with the URL of the `public` folder during the build.
+      Only files inside the `public` folder can be referenced from the HTML.
+
+      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+      work correctly both with client-side routing and a non-root public URL.
+      Learn how to configure a non-root public URL by running `npm run build`.
+    -->
+    <title>React App</title>
+  </head>
+  <body>
+    <noscript>
+      You need to enable JavaScript to run this app.
+    </noscript>
+    <div id="root"></div>
+    <!--
+      This HTML file is a template.
+      If you open it directly in the browser, you will see an empty page.
+
+      You can add webfonts, meta tags, or analytics to this file.
+      The build step will place the bundled scripts into the <body> tag.
+
+      To begin the development, run `npm start` or `yarn start`.
+      To create a production bundle, use `npm run build` or `yarn build`.
+    -->
+  </body>
+</html>

web/public/manifest.json

@@ -0,0 +1,15 @@
+{
+  "short_name": "React App",
+  "name": "Create React App Sample",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    }
+  ],
+  "start_url": "./index.html",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

web/src/App.css

@@ -0,0 +1,28 @@
+.App {
+  text-align: center;
+}
+
+.App-logo {
+  animation: App-logo-spin infinite 20s linear;
+  height: 80px;
+}
+
+.App-header {
+  background-color: #222;
+  height: 150px;
+  padding: 20px;
+  color: white;
+}
+
+.App-title {
+  font-size: 1.5em;
+}
+
+.App-intro {
+  font-size: large;
+}
+
+@keyframes App-logo-spin {
+  from { transform: rotate(0deg); }
+  to { transform: rotate(360deg); }
+}

web/src/App.js

@@ -0,0 +1,15 @@
+import React, { Component } from 'react';
+import DomainList from './components/DomainList'
+import './App.css';
+
+class App extends Component {
+  render() {
+    return (
+      <div className="App">
+        <DomainList></DomainList>
+      </div>
+    );
+  }
+}
+
+export default App;

web/src/App.test.js

@@ -0,0 +1,8 @@
+import React from 'react';
+import ReactDOM from 'react-dom';
+import App from './App';
+
+it('renders without crashing', () => {
+  const div = document.createElement('div');
+  ReactDOM.render(<App />, div);
+});

web/src/components/Domain.js

@@ -0,0 +1,12 @@
+import React, { Component } from 'react';
+
+class Domain extends Component {
+
+  render() {
+    return (
+        <p> {this.props.domain.domainName}.{this.props.domain.tld}</p>
+    );
+  }
+}
+
+export default Domain;

web/src/components/DomainList.js

@@ -0,0 +1,46 @@
+import React, { Component } from 'react';
+import {getDomains} from '../helpers/api';
+import Domain from './Domain';
+
+import ReactTable from "react-table";
+import "react-table/react-table.css";
+
+class DomainList extends Component {
+
+  constructor(props){
+    super(props);
+  
+    getDomains().then(l => l.text()).then(l => {
+      let parsedList = JSON.parse(l);
+      this.setState({domains:parsedList});
+    });
+
+    this.state = {domains:[]};
+  }
+
+  render() {
+    return (
+      <ReactTable
+        data = {this.state.domains}
+        columns={[
+          {
+            Header: 'Domain name',
+            accessor: 'domainName',
+          },
+          {
+            Header: 'TLD domain',
+            accessor: 'tld',
+          },
+          {
+            Header: 'Expiration date',
+            accessor: 'expirationDate',
+          }
+        ]}
+        defaultPageSize={10}
+        className="-striped -highlight"
+      />
+    )
+  }
+}
+
+export default DomainList;

web/src/config.js

@@ -0,0 +1 @@
+export const BASE_URL = 'localhost:4000';

web/src/helpers/api.js

@@ -0,0 +1,8 @@
+import fetch from 'isomorphic-fetch';
+import {BASE_URL} from '../config';
+
+export const getDomains = () => {
+    let url = `http://${BASE_URL}/list`
+  
+    return fetch(url)
+}

web/src/index.css

@@ -0,0 +1,5 @@
+body {
+  margin: 0;
+  padding: 0;
+  font-family: sans-serif;
+}

web/src/index.js

@@ -0,0 +1,8 @@
+import React from 'react';
+import ReactDOM from 'react-dom';
+import './index.css';
+import App from './App';
+import registerServiceWorker from './registerServiceWorker';
+
+ReactDOM.render(<App />, document.getElementById('root'));
+registerServiceWorker();

web/src/logo.svg

@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 841.9 595.3">
+    <g fill="#61DAFB">
+        <path d="M666.3 296.5c0-32.5-40.7-63.3-103.1-82.4 14.4-63.6 8-114.2-20.2-130.4-6.5-3.8-14.1-5.6-22.4-5.6v22.3c4.6 0 8.3.9 11.4 2.6 13.6 7.8 19.5 37.5 14.9 75.7-1.1 9.4-2.9 19.3-5.1 29.4-19.6-4.8-41-8.5-63.5-10.9-13.5-18.5-27.5-35.3-41.6-50 32.6-30.3 63.2-46.9 84-46.9V78c-27.5 0-63.5 19.6-99.9 53.6-36.4-33.8-72.4-53.2-99.9-53.2v22.3c20.7 0 51.4 16.5 84 46.6-14 14.7-28 31.4-41.3 49.9-22.6 2.4-44 6.1-63.6 11-2.3-10-4-19.7-5.2-29-4.7-38.2 1.1-67.9 14.6-75.8 3-1.8 6.9-2.6 11.5-2.6V78.5c-8.4 0-16 1.8-22.6 5.6-28.1 16.2-34.4 66.7-19.9 130.1-62.2 19.2-102.7 49.9-102.7 82.3 0 32.5 40.7 63.3 103.1 82.4-14.4 63.6-8 114.2 20.2 130.4 6.5 3.8 14.1 5.6 22.5 5.6 27.5 0 63.5-19.6 99.9-53.6 36.4 33.8 72.4 53.2 99.9 53.2 8.4 0 16-1.8 22.6-5.6 28.1-16.2 34.4-66.7 19.9-130.1 62-19.1 102.5-49.9 102.5-82.3zm-130.2-66.7c-3.7 12.9-8.3 26.2-13.5 39.5-4.1-8-8.4-16-13.1-24-4.6-8-9.5-15.8-14.4-23.4 14.2 2.1 27.9 4.7 41 7.9zm-45.8 106.5c-7.8 13.5-15.8 26.3-24.1 38.2-14.9 1.3-30 2-45.2 2-15.1 0-30.2-.7-45-1.9-8.3-11.9-16.4-24.6-24.2-38-7.6-13.1-14.5-26.4-20.8-39.8 6.2-13.4 13.2-26.8 20.7-39.9 7.8-13.5 15.8-26.3 24.1-38.2 14.9-1.3 30-2 45.2-2 15.1 0 30.2.7 45 1.9 8.3 11.9 16.4 24.6 24.2 38 7.6 13.1 14.5 26.4 20.8 39.8-6.3 13.4-13.2 26.8-20.7 39.9zm32.3-13c5.4 13.4 10 26.8 13.8 39.8-13.1 3.2-26.9 5.9-41.2 8 4.9-7.7 9.8-15.6 14.4-23.7 4.6-8 8.9-16.1 13-24.1zM421.2 430c-9.3-9.6-18.6-20.3-27.8-32 9 .4 18.2.7 27.5.7 9.4 0 18.7-.2 27.8-.7-9 11.7-18.3 22.4-27.5 32zm-74.4-58.9c-14.2-2.1-27.9-4.7-41-7.9 3.7-12.9 8.3-26.2 13.5-39.5 4.1 8 8.4 16 13.1 24 4.7 8 9.5 15.8 14.4 23.4zM420.7 163c9.3 9.6 18.6 20.3 27.8 32-9-.4-18.2-.7-27.5-.7-9.4 0-18.7.2-27.8.7 9-11.7 18.3-22.4 27.5-32zm-74 58.9c-4.9 7.7-9.8 15.6-14.4 23.7-4.6 8-8.9 16-13 24-5.4-13.4-10-26.8-13.8-39.8 13.1-3.1 26.9-5.8 41.2-7.9zm-90.5 125.2c-35.4-15.1-58.3-34.9-58.3-50.6 0-15.7 22.9-35.6 58.3-50.6 8.6-3.7 18-7 27.7-10.1 5.7 19.6 13.2 40 22.5 60.9-9.2 20.8-16.6 41.1-22.2 60.6-9.9-3.1-19.3-6.5-28-10.2zM310 490c-13.6-7.8-19.5-37.5-14.9-75.7 1.1-9.4 2.9-19.3 5.1-29.4 19.6 4.8 41 8.5 63.5 10.9 13.5 18.5 27.5 35.3 41.6 50-32.6 30.3-63.2 46.9-84 46.9-4.5-.1-8.3-1-11.3-2.7zm237.2-76.2c4.7 38.2-1.1 67.9-14.6 75.8-3 1.8-6.9 2.6-11.5 2.6-20.7 0-51.4-16.5-84-46.6 14-14.7 28-31.4 41.3-49.9 22.6-2.4 44-6.1 63.6-11 2.3 10.1 4.1 19.8 5.2 29.1zm38.5-66.7c-8.6 3.7-18 7-27.7 10.1-5.7-19.6-13.2-40-22.5-60.9 9.2-20.8 16.6-41.1 22.2-60.6 9.9 3.1 19.3 6.5 28.1 10.2 35.4 15.1 58.3 34.9 58.3 50.6-.1 15.7-23 35.6-58.4 50.6zM320.8 78.4z"/>
+        <circle cx="420.9" cy="296.5" r="45.7"/>
+        <path d="M520.5 78.1z"/>
+    </g>
+</svg>

web/src/registerServiceWorker.js

@@ -0,0 +1,108 @@
+// In production, we register a service worker to serve assets from local cache.
+
+// This lets the app load faster on subsequent visits in production, and gives
+// it offline capabilities. However, it also means that developers (and users)
+// will only see deployed updates on the "N+1" visit to a page, since previously
+// cached resources are updated in the background.
+
+// To learn more about the benefits of this model, read https://goo.gl/KwvDNy.
+// This link also includes instructions on opting out of this behavior.
+
+const isLocalhost = Boolean(
+  window.location.hostname === 'localhost' ||
+    // [::1] is the IPv6 localhost address.
+    window.location.hostname === '[::1]' ||
+    // 127.0.0.1/8 is considered localhost for IPv4.
+    window.location.hostname.match(
+      /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/
+    )
+);
+
+export default function register() {
+  if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) {
+    // The URL constructor is available in all browsers that support SW.
+    const publicUrl = new URL(process.env.PUBLIC_URL, window.location);
+    if (publicUrl.origin !== window.location.origin) {
+      // Our service worker won't work if PUBLIC_URL is on a different origin
+      // from what our page is served on. This might happen if a CDN is used to
+      // serve assets; see https://github.com/facebookincubator/create-react-app/issues/2374
+      return;
+    }
+
+    window.addEventListener('load', () => {
+      const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`;
+
+      if (isLocalhost) {
+        // This is running on localhost. Lets check if a service worker still exists or not.
+        checkValidServiceWorker(swUrl);
+      } else {
+        // Is not local host. Just register service worker
+        registerValidSW(swUrl);
+      }
+    });
+  }
+}
+
+function registerValidSW(swUrl) {
+  navigator.serviceWorker
+    .register(swUrl)
+    .then(registration => {
+      registration.onupdatefound = () => {
+        const installingWorker = registration.installing;
+        installingWorker.onstatechange = () => {
+          if (installingWorker.state === 'installed') {
+            if (navigator.serviceWorker.controller) {
+              // At this point, the old content will have been purged and
+              // the fresh content will have been added to the cache.
+              // It's the perfect time to display a "New content is
+              // available; please refresh." message in your web app.
+              console.log('New content is available; please refresh.');
+            } else {
+              // At this point, everything has been precached.
+              // It's the perfect time to display a
+              // "Content is cached for offline use." message.
+              console.log('Content is cached for offline use.');
+            }
+          }
+        };
+      };
+    })
+    .catch(error => {
+      console.error('Error during service worker registration:', error);
+    });
+}
+
+function checkValidServiceWorker(swUrl) {
+  // Check if the service worker can be found. If it can't reload the page.
+  fetch(swUrl)
+    .then(response => {
+      // Ensure service worker exists, and that we really are getting a JS file.
+      if (
+        response.status === 404 ||
+        response.headers.get('content-type').indexOf('javascript') === -1
+      ) {
+        // No service worker found. Probably a different app. Reload the page.
+        navigator.serviceWorker.ready.then(registration => {
+          registration.unregister().then(() => {
+            window.location.reload();
+          });
+        });
+      } else {
+        // Service worker found. Proceed as normal.
+        registerValidSW(swUrl);
+      }
+    })
+    .catch(() => {
+      console.log(
+        'No internet connection found. App is running in offline mode.'
+      );
+    });
+}
+
+export function unregister() {
+  if ('serviceWorker' in navigator) {
+    navigator.serviceWorker.ready.then(registration => {
+      registration.unregister();
+    });
+  }
+}