From 87134e4edefdb287aae937341cd9ee4df36d1c73 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 31 May 2019 05:42:50 +0200 Subject: [PATCH] display upload errors, unknown members --- .../UploadDLockData/components/FileUpload.js | 9 +- .../components/UploadResults.js | 83 +++++---- client/src/store/reducers/doorLockReducers.js | 2 +- constants/constants.js | 6 + controllers/doorLock.js | 10 +- helpers/api.js | 12 ++ .../20190529103954-create-door-lock-events.js | 3 +- models/DoorLockEvent.js | 3 +- package-lock.json | 30 +++- package.json | 1 + services/doorLock.js | 162 ++++++++++-------- services/officeRnD/members.js | 33 ++++ 12 files changed, 246 insertions(+), 108 deletions(-) create mode 100644 helpers/api.js create mode 100644 services/officeRnD/members.js diff --git a/client/src/scenes/UploadDLockData/components/FileUpload.js b/client/src/scenes/UploadDLockData/components/FileUpload.js index d0561ff..b185805 100644 --- a/client/src/scenes/UploadDLockData/components/FileUpload.js +++ b/client/src/scenes/UploadDLockData/components/FileUpload.js @@ -31,6 +31,7 @@ class FileUpload extends Component { }; render() { + const { pending } = this.props; return (
- Upload + Upload
); } } +const mapStateToProps = (state) => ({ + pending: state.doorLockData.pending, +}); + const mapDispatchToProps = (dispatch) => ({ uploadDoorLockData: (doorLockDataFile) => uploadDoorLockData(dispatch, doorLockDataFile) }); -export default connect(null, mapDispatchToProps)(FileUpload); +export default connect(mapStateToProps, mapDispatchToProps)(FileUpload); diff --git a/client/src/scenes/UploadDLockData/components/UploadResults.js b/client/src/scenes/UploadDLockData/components/UploadResults.js index cc688a5..6a6bbfa 100644 --- a/client/src/scenes/UploadDLockData/components/UploadResults.js +++ b/client/src/scenes/UploadDLockData/components/UploadResults.js @@ -1,13 +1,58 @@ import React, { Component } from 'react'; import { connect } from 'react-redux'; -import { Loader, Message } from 'semantic-ui-react'; +import { Loader, Message, Tab, Label, Menu } from 'semantic-ui-react'; class UploadResults extends Component { render(){ const {pending, result, error} = this.props; - const parserErrors = result && result.parserErrors && result.parserErrors.length > 0 ? result.parserErrors : null; - const parsedDataCount = result && result.parsedData && result.parsedData.length > 0 ? result.parsedData.length : null; + const parsedEntries = result && result.parsedData ? result.parsedData : []; + const errorEntries = result && result.parserErrors ? result.parserErrors : []; + const unknownMembers = result && result.unknownMembers ? result.unknownMembers : []; + + const renderParsedEntriesTab = () => { + return ( +
+
+ +

{parsedEntries.length} entries successfully parsed

+ +
+ ); + }; + + const renderErrorTabResults = (results) => { + return ( +
+
+ { + results.map((entry, index) => { + return ( +
+
+ + {entry.error} +

{JSON.stringify(entry.details)}

+

File : {entry.file}

+ +
+ ); + }) + } +
+ ); + }; + + const parsedEntriesTabTitle = (Parsed Entries); + const errorEntriesTabTitle = (Error Entries); + const unknownMembersTabTitle = (Unknown Members); + + const panes = [ + {menuItem: parsedEntriesTabTitle, render: renderParsedEntriesTab}, + {menuItem: errorEntriesTabTitle, render: () => renderErrorTabResults(errorEntries)}, + {menuItem: unknownMembersTabTitle, render: () => renderErrorTabResults(unknownMembers)} + ]; + return (
@@ -16,36 +61,16 @@ class UploadResults extends Component { }
{ - error && - - Upload failed -

There was error uploading file

- + !pending && !error && result && + } -
{ - !error && parsedDataCount && - - Upload complete -

{parsedDataCount} entries successfully inserted

- {parserErrors &&

Some entries could not be parsed. Details are shown below

} + !pending && error && + + Upload Failed +

{error.data}

 } -
- { - !error && parserErrors && parserErrors.map((parserError, index) => { - return ( -
- - {parserError.error} -

{JSON.stringify(parserError.details)}

-

File : {parserError.file}

- -
-
- ); - }) - }
) } diff --git a/client/src/store/reducers/doorLockReducers.js b/client/src/store/reducers/doorLockReducers.js index 58cc7d0..97158d3 100644 --- a/client/src/store/reducers/doorLockReducers.js +++ b/client/src/store/reducers/doorLockReducers.js @@ -6,7 +6,7 @@ import { const initialState = { pending: false, - result: {}, + result: null, error: null, }; diff --git a/constants/constants.js b/constants/constants.js index 4de18b5..77fc7f8 100644 --- a/constants/constants.js +++ b/constants/constants.js @@ -11,6 +11,11 @@ const csvParserErrors = { UNKNOWN_COLUMN: 'Unknown column', INVALID_ENTRY_EXPECTED_USER: 'Invalid entry type. Expected user entry type', INVALID_ENTRY_EXPECTED_PASSAGE_MODE: 'Invalid entry type. Expected enable/disable passage mode', + UNKNOWN_MEMBER: 'Member is not registered in OfficeRnD system', +}; + +const officeRnDAPIErrors = { + FAILED_TO_FETCH_MEMBERS: 'Failed to fetch members', }; module.exports = { @@ -21,4 +26,5 @@ module.exports = { USER_LOCKED_DOOR, USER_UNLOCKED_DOOR, csvParserErrors, + officeRnDAPIErrors, }; diff --git a/controllers/doorLock.js b/controllers/doorLock.js index 7925263..c7bdf5e 100644 --- a/controllers/doorLock.js +++ b/controllers/doorLock.js @@ -1,10 +1,10 @@ 'use strict'; const { parseDoorLockDataFile, writeDoorLockEvent } = require("../services/doorLock"); +const { officeRnDAPIErrors } = require('../constants/constants'); const IncomingForm = require('formidable').IncomingForm; - const uploadDoorLockData = (req, res) => { const form = new IncomingForm(); const parsingResults = []; @@ -20,24 +20,26 @@ const uploadDoorLockData = (req, res) => { .then((parserResults) => { const parsedData = []; const parserErrors = []; + const unknownMembers = []; parserResults.forEach((parserResult) => { parsedData.push(...parserResult.parsedData); parserErrors.push(...parserResult.errors); + unknownMembers.push(...parserResult.unknownMembers); }); res.json({ parsedData, parserErrors, + unknownMembers }); parsedData.forEach((entry) => { writeDoorLockEvent(entry); }); }) - .catch((error) => { - console.log(error); - res.json({result: 'error'}); + .catch(() => { + res.status(500).send(officeRnDAPIErrors.FAILED_TO_FETCH_MEMBERS); }); }); diff --git a/helpers/api.js b/helpers/api.js new file mode 100644 index 0000000..441105d --- /dev/null +++ b/helpers/api.js @@ -0,0 +1,12 @@ +const axios = require('axios'); + +const officeRnDToken = process.env.OFFICE_RnD_TOKEN; + +const API = axios.create({ + baseURL: 'https://app.officernd.com/api/v1/organizations/sima-space-test-environment', + headers: {'Authorization': `Bearer ${officeRnDToken}`} +}); + +module.exports = { + API, +}; diff --git a/migrations/20190529103954-create-door-lock-events.js b/migrations/20190529103954-create-door-lock-events.js index 93990bd..6b8a452 100644 --- a/migrations/20190529103954-create-door-lock-events.js +++ b/migrations/20190529103954-create-door-lock-events.js @@ -9,8 +9,9 @@ module.exports = { primaryKey: true, type: Sequelize.INTEGER }, - memberName: Sequelize.STRING, + memberName: Sequelize.TEXT, memberNumber: Sequelize.INTEGER, + memberId: Sequelize.TEXT, event: { type: Sequelize.ENUM, values: ['locked', 'unlocked'] diff --git a/models/DoorLockEvent.js b/models/DoorLockEvent.js index f159dff..f07cc5a 100644 --- a/models/DoorLockEvent.js +++ b/models/DoorLockEvent.js @@ -4,8 +4,9 @@ const { USER_LOCKED_DOOR, USER_UNLOCKED_DOOR } = require('../constants/constants module.exports = (sequelize, DataTypes) => { const doorLockEvent = sequelize.define('doorLockEvent', { - memberName: DataTypes.STRING, + memberName: DataTypes.TEXT, memberNumber: DataTypes.INTEGER, + memberId: DataTypes.TEXT, event: { type: DataTypes.ENUM, values: [USER_LOCKED_DOOR, USER_UNLOCKED_DOOR] diff --git a/package-lock.json b/package-lock.json index c2b51d6..85b4ea6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -119,6 +119,15 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, + "axios": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", + "requires": { + "follow-redirects": "^1.3.0", + "is-buffer": "^1.1.5" + } + }, "babel-runtime": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", @@ -1118,6 +1127,24 @@ "locate-path": "^3.0.0" } }, + "follow-redirects": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.7.0.tgz", + "integrity": "sha512-m/pZQy4Gj287eNy94nivy5wchN3Kp+Q5WgUPNy5lJSZ3sgkVKSYV/ZChMAQVIgx1SqfZ2zBZtPA2YlXIWxxJOQ==", + "requires": { + "debug": "^3.2.6" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -1955,8 +1982,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-ci": { "version": "1.2.1", diff --git a/package.json b/package.json index 7e736b1..7a8ddbb 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "node": "11.12.x" }, "dependencies": { + "axios": "^0.18.0", "csv-parser": "^2.3.0", "dotenv": "^8.0.0", "express": "^4.17.0", diff --git a/services/doorLock.js b/services/doorLock.js index c2552b7..30a7345 100644 --- a/services/doorLock.js +++ b/services/doorLock.js @@ -14,84 +14,110 @@ const { csvParserErrors, } = require('../constants/constants'); +const { fetchAllMembers, findMember } = require('../services/officeRnD/members'); + const parseDoorLockDataFile = (file) => { return new Promise ((resolve, reject) => { const results = []; const errors = []; + const unknownMembers = []; let isValidFile = true; - fs.createReadStream(file.path) - .pipe(csv({ - mapHeaders: ({ header, index }) => header.trim().toLowerCase(), - mapValues: ({ header, index, value }) => value.trim() - })) - .on('headers', (headers) => { - headers.forEach((header) => { - if (!VALID_CSV_HEADERS.includes(header.trim())){ - isValidFile = false; - console.log('INVALID HEADER'); - errors.push({ - error: csvParserErrors.UNKNOWN_COLUMN, - details: header, - file: file.name, + fetchAllMembers() + .then(() => { + fs.createReadStream(file.path) + .pipe(csv({ + mapHeaders: ({ header, index }) => header.trim().toLowerCase(), + mapValues: ({ header, index, value }) => value.trim() + })) + .on('headers', (headers) => { + headers.forEach((header) => { + if (!VALID_CSV_HEADERS.includes(header.trim())){ + isValidFile = false; + console.log('INVALID HEADER'); + errors.push({ + error: csvParserErrors.UNKNOWN_COLUMN, + details: header, + file: file.name, + }); + } }); - } - }); - }) - .on('data', (data) => { - if (!isValidFile) { - return; - } - - const eventType = data.event.trim(); - if ([USER_ENTRY_EVENT, ENABLE_PASSAGE_MODE, DISABLE_PASSAGE_MODE].includes(eventType)){ - results.push(data); - } - }) - .on('end', () => { - const parsedData = []; - let i = 0; - while (i < results.length){ - //Verify pair - //First entry should be user entry and second should be enable / disable passage - const firstEntry = results[i]; - const secondEntry = results[i+1]; - - - if (firstEntry && (firstEntry.event === USER_ENTRY_EVENT)){ - if (secondEntry && (secondEntry.event === ENABLE_PASSAGE_MODE || secondEntry.event === DISABLE_PASSAGE_MODE)){ - const event = (secondEntry.event === ENABLE_PASSAGE_MODE) ? USER_UNLOCKED_DOOR : USER_LOCKED_DOOR; - const entryData = { - memberName: firstEntry.name, - memberNumber: firstEntry['user no'], - date: firstEntry.date, - time: firstEntry.time, - event, - }; - parsedData.push(entryData); - i+=2; - } else { - errors.push({ - error: csvParserErrors.INVALID_ENTRY_EXPECTED_PASSAGE_MODE, - details: secondEntry || 'Last row in file', - file: file.name, - }); - i+=1; + }) + .on('data', (data) => { + if (!isValidFile) { + return; } - } else { - errors.push({ - error: csvParserErrors.INVALID_ENTRY_EXPECTED_USER, - details: firstEntry, - file: file.name, + + const eventType = data.event.trim(); + if ([USER_ENTRY_EVENT, ENABLE_PASSAGE_MODE, DISABLE_PASSAGE_MODE].includes(eventType)){ + results.push(data); + } + }) + .on('end', () => { + const parsedData = []; + let i = 0; + while (i < results.length){ + //Verify pair + //First entry should be user entry and second should be enable / disable passage + const firstEntry = results[i]; + const secondEntry = results[i+1]; + + if (firstEntry && (firstEntry.event === USER_ENTRY_EVENT)){ + if (secondEntry && (secondEntry.event === ENABLE_PASSAGE_MODE || secondEntry.event === DISABLE_PASSAGE_MODE)){ + const event = (secondEntry.event === ENABLE_PASSAGE_MODE) ? USER_UNLOCKED_DOOR : USER_LOCKED_DOOR; + const memberObject = findMember(firstEntry.name); + + //Verify that member is registered in OfficeRnD system + if (memberObject){ + const entryData = { + memberName: firstEntry.name, + memberNumber: firstEntry['user no'], + memberId: memberObject.memberId, + date: firstEntry.date, + time: firstEntry.time, + event, + }; + + parsedData.push(entryData); + } else { + //Check if member is already labeled as unknown + const unknownMember = unknownMembers.find((member) => member.details === firstEntry.name); + if (!unknownMember){ + unknownMembers.push({ + error: csvParserErrors.UNKNOWN_MEMBER, + details: firstEntry.name, + file: file.name, + }); + } + } + i+=2; + } else { + errors.push({ + error: csvParserErrors.INVALID_ENTRY_EXPECTED_PASSAGE_MODE, + details: secondEntry || 'Last row in file', + file: file.name, + }); + i+=1; + } + } else { + errors.push({ + error: csvParserErrors.INVALID_ENTRY_EXPECTED_USER, + details: firstEntry, + file: file.name, + }); + i+=1; + } + } + resolve({ + parsedData, + unknownMembers, + errors }); - i+=1; - } - } - resolve({ - parsedData, - errors - }); + }); + }) + .catch((error) => { + reject(error); }); }); }; diff --git a/services/officeRnD/members.js b/services/officeRnD/members.js new file mode 100644 index 0000000..e45b776 --- /dev/null +++ b/services/officeRnD/members.js @@ -0,0 +1,33 @@ +'use strict'; + +const { API } = require('../../helpers/api'); + +const membersList = []; + +const fetchAllMembers = () => { + return new Promise((resolve, reject) => { + API.get('/members') + .then((result) => { + const members = result.data || []; + members.forEach((member) => { + membersList.push({ + name: member.name, + memberId: member['_id'], + }); + }); + resolve(); + }) + .catch((error) => { + reject(error); + }); + }); +}; + +const findMember = (memberName) => { + return membersList.find((member) => member.name === memberName); +}; + +module.exports = { + fetchAllMembers, + findMember, +};