fix parsing reports, change tables to use timestamps

constants/constants.js

@@ -8,9 +8,9 @@ const USER_UNLOCKED_DOOR = 'unlocked';
 const VALID_CSV_HEADERS = ['Date', 'Time', 'User No', 'Name', 'Event'];
 
 const csvParserErrors = {
-    UNKNOWN_COLUMN: 'Unknown column',
+    INVALID_HEADERS: 'Invalid headers',
     INVALID_ENTRY_EXPECTED_USER: 'Invalid entry type. Expected user entry type',
-    INVALID_ENTRY_EXPECTED_PASSAGE_MODE: 'Invalid entry type. Expected enable/disable passage mode',
+    INVALID_ENTRY_EXPECTED_PASSAGE_MODE: 'Invalid entry type. Expected enable/disable passage mode following user entry',
     UNKNOWN_MEMBER: 'Member is not registered in OfficeRnD system',
 };
 

migrations/20190529103954-create-door-lock-events.js

@@ -16,8 +16,7 @@ module.exports = {
         type: Sequelize.ENUM,
         values: ['locked', 'unlocked']
       },
-      date: Sequelize.DATEONLY,
+      timestamp: Sequelize.DATE,
-      time: Sequelize.TIME,
       createdAt: {
         allowNull: false,
         type: Sequelize.DATE

migrations/20190531154129-door-lock-incidents.js

@@ -0,0 +1,40 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.createTable('doorLockIncidents', {
+      id: {
+        allowNull: false,
+        autoIncrement: true,
+        primaryKey: true,
+        type: Sequelize.INTEGER
+      },
+      reservationId: Sequelize.TEXT,
+      memberId: Sequelize.TEXT,
+      resource: Sequelize.TEXT,
+      bookingStart: Sequelize.DATE,
+      bookingEnd: Sequelize.DATE,
+      doorLockEventTimestamp: Sequelize.DATE,
+      doorLockEventType: {
+        type: Sequelize.ENUM,
+        values: ['locked', 'unlocked']
+      },
+      chargeType: {
+        type: Sequelize.ENUM,
+        values: ['unlocked', 'unscheduled']
+      },
+      chargeFee: Sequelize.FLOAT,
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      }
+    });
+  },
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.dropTable('doorLockIncidents');
+  }
+};

models/doorLockEvent.js

@@ -11,8 +11,7 @@ module.exports = (sequelize, DataTypes) => {
             type: DataTypes.ENUM,
             values: [USER_LOCKED_DOOR, USER_UNLOCKED_DOOR]
         },
-        date: DataTypes.DATEONLY,
+        timestamp: DataTypes.DATE,
-        time: DataTypes.TIME,
     }, {});
     doorLockEvent.associate = function(models) {
         // associations can be defined here

package.json

@@ -10,7 +10,7 @@
     "docker-build": "docker build -t simaspace .",
     "docker-start": "docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=CrmIntegration --name pg_simaspace -d -p 5432:5432 simaspace",
     "docker-stop": "docker stop pg_simaspace",
-    "setup": "npm run install-server && npm run install-client && npm run docker-build && npm run docker-start && sleep 2 && npm run migrate",
+    "setup": "npm run install-server && npm run install-client && npm run docker-build && npm run docker-start && sleep 5 && npm run migrate",
     "migrate": "npx sequelize db:migrate",
     "start-server": "nodemon server.js",
     "start-client": "cd client && yarn start",
@@ -27,6 +27,7 @@
     "express": "^4.17.0",
     "express-basic-auth": "^1.2.0",
     "formidable": "^1.2.1",
+    "moment": "^2.24.0",
     "pg": "^7.11.0",
     "sequelize": "^5.8.6",
     "sequelize-cli": "^5.4.0"

services/doorLock.js

@@ -3,6 +3,7 @@
 const db = require('../models/index');
 const fs = require('fs');
 const csv = require('csv-parser');
+const moment = require('moment');
 
 const {
     USER_ENTRY_EVENT,
@@ -32,17 +33,28 @@ const parseDoorLockDataFile = (file) => {
                         mapValues: ({ header, index, value }) => value.trim()
                     }))
                     .on('headers', (headers) => {
-                        headers.forEach((header) => {
+
-                            if (!VALID_CSV_HEADERS.includes(header.trim())){
+                        const sortedValidHeadersArray = VALID_CSV_HEADERS.concat().sort();
-                                isValidFile = false;
+                        const sortedParsedHeadersArray = headers.map(header => header.trim()).sort();
-                                console.log('INVALID HEADER');
+
-                                errors.push({
+                        let validHeaders = true;
-                                    error: csvParserErrors.UNKNOWN_COLUMN,
+                        if (sortedParsedHeadersArray.length !== sortedValidHeadersArray.length) {
-                                    details: header,
+                            validHeaders = false;
-                                    file: file.name,
+                        }else {
-                                });
+                            for (let i = 0; i < sortedValidHeadersArray.length; i++){
+                                validHeaders = validHeaders
+                                    && (sortedValidHeadersArray[i] === sortedParsedHeadersArray[i]);
                             }
-                        });
+                        }
+
+                        if (!validHeaders){
+                            isValidFile = false;
+                            errors.push({
+                                error: csvParserErrors.INVALID_HEADERS,
+                                details: `Expected headers : ${JSON.stringify(VALID_CSV_HEADERS)}`,
+                                file: file.name,
+                            });
+                        }
                     })
                     .on('data', (data) => {
                         if (!isValidFile) {
@@ -59,14 +71,29 @@ const parseDoorLockDataFile = (file) => {
                         let i = 0;
                         while (i < results.length){
                             //Verify pair
-                            //First entry should be user entry and second should be enable / disable passage
+                            //First entry type should be user entry and second should be enable / disable passage
                             const firstEntry = results[i];
                             const secondEntry = results[i+1];
 
                             if (firstEntry && (firstEntry.event === USER_ENTRY_EVENT)){
+                                const memberObject = findMember(firstEntry.name);
+
+                                if (!memberObject){
+                                    //Check if member is already labeled as unknown
+                                    const unknownMember = unknownMembers.find((member) => member.details === firstEntry.name);
+                                    if (!unknownMember){
+                                        unknownMembers.push({
+                                            error: csvParserErrors.UNKNOWN_MEMBER,
+                                            details: firstEntry.name,
+                                            file: file.name,
+                                        });
+                                    }
+                                }
+
                                 if (secondEntry && (secondEntry.event === ENABLE_PASSAGE_MODE || secondEntry.event === DISABLE_PASSAGE_MODE)){
                                     const event = (secondEntry.event === ENABLE_PASSAGE_MODE) ? USER_UNLOCKED_DOOR : USER_LOCKED_DOOR;
-                                    const memberObject = findMember(firstEntry.name);
+                                    const dateTimeString = `${firstEntry.date} ${firstEntry.time}`;
+                                    const timestamp = moment.utc(dateTimeString, 'MM/DD/YY HH:mm:ss A').toISOString();
 
                                     //Verify that member is registered in OfficeRnD system
                                     if (memberObject){
@@ -74,28 +101,17 @@ const parseDoorLockDataFile = (file) => {
                                             memberName: firstEntry.name,
                                             memberNumber: firstEntry['user no'],
                                             memberId: memberObject.memberId,
-                                            date: firstEntry.date,
+                                            timestamp,
-                                            time: firstEntry.time,
                                             event,
                                         };
 
                                         parsedData.push(entryData);
-                                    } else {
-                                        //Check if member is already labeled as unknown
-                                        const unknownMember = unknownMembers.find((member) => member.details === firstEntry.name);
-                                        if (!unknownMember){
-                                            unknownMembers.push({
-                                                error: csvParserErrors.UNKNOWN_MEMBER,
-                                                details: firstEntry.name,
-                                                file: file.name,
-                                            });
-                                        }
                                     }
                                     i+=2;
                                 } else {
                                     errors.push({
                                         error: csvParserErrors.INVALID_ENTRY_EXPECTED_PASSAGE_MODE,
-                                        details: secondEntry || 'Last row in file',
+                                        details: firstEntry,
                                         file: file.name,
                                     });
                                     i+=1;