senaduka/old-psihologija
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

display upload errors, unknown members

Browse Source
This commit is contained in:
Bilal Catic
2019-05-31 05:42:50 +02:00
parent 48503de853
commit 87134e4ede
12 changed files with 246 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
client/src/scenes/UploadDLockData/components/FileUpload.js
View File

@@ -31,6 +31,7 @@ class FileUpload extends Component {
};
render() {
const { pending } = this.props;
return (
<div>
<Form.Input
@@ -41,14 +42,18 @@ class FileUpload extends Component {
accept=".csv"
onChange={this.onFileChange}
/>
<Form.Button onClick={this.onUploadClick}>Upload</Form.Button>
<Form.Button onClick={this.onUploadClick} disabled={pending} >Upload</Form.Button>
</div>
);
}
}
const mapStateToProps = (state) => ({
pending: state.doorLockData.pending,
});
const mapDispatchToProps = (dispatch) => ({
uploadDoorLockData: (doorLockDataFile) => uploadDoorLockData(dispatch, doorLockDataFile)
});
export default connect(null, mapDispatchToProps)(FileUpload);
export default connect(mapStateToProps, mapDispatchToProps)(FileUpload);

83
client/src/scenes/UploadDLockData/components/UploadResults.js
View File

@@ -1,13 +1,58 @@
import React, { Component } from 'react';
import { connect } from 'react-redux';
import { Loader, Message } from 'semantic-ui-react';
import { Loader, Message, Tab, Label, Menu } from 'semantic-ui-react';
class UploadResults extends Component {
render(){
const {pending, result, error} = this.props;
const parserErrors = result && result.parserErrors && result.parserErrors.length > 0 ? result.parserErrors : null;
const parsedDataCount = result && result.parsedData && result.parsedData.length > 0 ? result.parsedData.length : null;
const parsedEntries = result && result.parsedData ? result.parsedData : [];
const errorEntries = result && result.parserErrors ? result.parserErrors : [];
const unknownMembers = result && result.unknownMembers ? result.unknownMembers : [];
const renderParsedEntriesTab = () => {
return (
<div>
<br/>
<Message positive>
<p>{parsedEntries.length} entries successfully parsed</p>
</Message>
</div>
);
};
const renderErrorTabResults = (results) => {
return (
<div>
<br/>
{
results.map((entry, index) => {
return (
<div key={`error-${entry.error}-${index}`}>
<br/>
<Message negative>
<Message.Header>{entry.error}</Message.Header>
<p>{JSON.stringify(entry.details)}</p>
<p>File : {entry.file}</p>
</Message>
</div>
);
})
}
</div>
);
};
const parsedEntriesTabTitle = (<Menu.Item key="parsed-entries">Parsed Entries<Label>{parsedEntries.length}</Label></Menu.Item>);
const errorEntriesTabTitle = (<Menu.Item key="error-entries">Error Entries<Label>{errorEntries.length}</Label></Menu.Item>);
const unknownMembersTabTitle = (<Menu.Item key="unknown-members">Unknown Members<Label>{unknownMembers.length}</Label></Menu.Item>);
const panes = [
{menuItem: parsedEntriesTabTitle, render: renderParsedEntriesTab},
{menuItem: errorEntriesTabTitle, render: () => renderErrorTabResults(errorEntries)},
{menuItem: unknownMembersTabTitle, render: () => renderErrorTabResults(unknownMembers)}
];
return (
<div>
@@ -16,36 +61,16 @@ class UploadResults extends Component {
}
<br/>
{
error &&
<Message negative>
<Message.Header>Upload failed</Message.Header>
<p>There was error uploading file</p>
</Message>
!pending && !error && result &&
<Tab panes={panes}/>
}
<br/>
{
!error && parsedDataCount &&
<Message positive>
<Message.Header>Upload complete</Message.Header>
<p>{parsedDataCount} entries successfully inserted</p>
{parserErrors && <p style={{ color: 'red' }}>Some entries could not be parsed. Details are shown below</p>}
!pending && error &&
<Message>
<Message.Header>Upload Failed</Message.Header>
<p>{error.data}</p>
</Message>
}
<br/>
{
!error && parserErrors && parserErrors.map((parserError, index) => {
return (
<div key={`message-${index}`}>
<Message negative>
<Message.Header>{parserError.error}</Message.Header>
<p>{JSON.stringify(parserError.details)}</p>
<p>File : {parserError.file}</p>
</Message>
<br/>
</div>
);
})
}
</div>
)
}

2
client/src/store/reducers/doorLockReducers.js
View File

@@ -6,7 +6,7 @@ import {
const initialState = {
pending: false,
result: {},
result: null,
error: null,
};

6
constants/constants.js
View File

@@ -11,6 +11,11 @@ const csvParserErrors = {
UNKNOWN_COLUMN: 'Unknown column',
INVALID_ENTRY_EXPECTED_USER: 'Invalid entry type. Expected user entry type',
INVALID_ENTRY_EXPECTED_PASSAGE_MODE: 'Invalid entry type. Expected enable/disable passage mode',
UNKNOWN_MEMBER: 'Member is not registered in OfficeRnD system',
};
const officeRnDAPIErrors = {
FAILED_TO_FETCH_MEMBERS: 'Failed to fetch members',
};
module.exports = {
@@ -21,4 +26,5 @@ module.exports = {
USER_LOCKED_DOOR,
USER_UNLOCKED_DOOR,
csvParserErrors,
officeRnDAPIErrors,
};

10
controllers/doorLock.js
View File

@@ -1,10 +1,10 @@
'use strict';
const { parseDoorLockDataFile, writeDoorLockEvent } = require("../services/doorLock");
const { officeRnDAPIErrors } = require('../constants/constants');
const IncomingForm = require('formidable').IncomingForm;
const uploadDoorLockData = (req, res) => {
const form = new IncomingForm();
const parsingResults = [];
@@ -20,24 +20,26 @@ const uploadDoorLockData = (req, res) => {
.then((parserResults) => {
const parsedData = [];
const parserErrors = [];
const unknownMembers = [];
parserResults.forEach((parserResult) => {
parsedData.push(...parserResult.parsedData);
parserErrors.push(...parserResult.errors);
unknownMembers.push(...parserResult.unknownMembers);
});
res.json({
parsedData,
parserErrors,
unknownMembers
});
parsedData.forEach((entry) => {
writeDoorLockEvent(entry);
});
})
.catch((error) => {
console.log(error);
res.json({result: 'error'});
.catch(() => {
res.status(500).send(officeRnDAPIErrors.FAILED_TO_FETCH_MEMBERS);
});
});

12
helpers/api.js Normal file
View File

@@ -0,0 +1,12 @@
const axios = require('axios');
const officeRnDToken = process.env.OFFICE_RnD_TOKEN;
const API = axios.create({
baseURL: 'https://app.officernd.com/api/v1/organizations/sima-space-test-environment',
headers: {'Authorization': `Bearer ${officeRnDToken}`}
});
module.exports = {
API,
};

3
migrations/20190529103954-create-door-lock-events.js
View File

@@ -9,8 +9,9 @@ module.exports = {
primaryKey: true,
type: Sequelize.INTEGER
},
memberName: Sequelize.STRING,
memberName: Sequelize.TEXT,
memberNumber: Sequelize.INTEGER,
memberId: Sequelize.TEXT,
event: {
type: Sequelize.ENUM,
values: ['locked', 'unlocked']

3
models/DoorLockEvent.js
View File

@@ -4,8 +4,9 @@ const { USER_LOCKED_DOOR, USER_UNLOCKED_DOOR } = require('../constants/constants
module.exports = (sequelize, DataTypes) => {
const doorLockEvent = sequelize.define('doorLockEvent', {
memberName: DataTypes.STRING,
memberName: DataTypes.TEXT,
memberNumber: DataTypes.INTEGER,
memberId: DataTypes.TEXT,
event: {
type: DataTypes.ENUM,
values: [USER_LOCKED_DOOR, USER_UNLOCKED_DOOR]

30
package-lock.json generated
View File

@@ -119,6 +119,15 @@
"integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
"dev": true
},
"axios": {
"version": "0.18.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz",
"integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=",
"requires": {
"follow-redirects": "^1.3.0",
"is-buffer": "^1.1.5"
}
},
"babel-runtime": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
@@ -1118,6 +1127,24 @@
"locate-path": "^3.0.0"
}
},
"follow-redirects": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.7.0.tgz",
"integrity": "sha512-m/pZQy4Gj287eNy94nivy5wchN3Kp+Q5WgUPNy5lJSZ3sgkVKSYV/ZChMAQVIgx1SqfZ2zBZtPA2YlXIWxxJOQ==",
"requires": {
"debug": "^3.2.6"
},
"dependencies": {
"debug": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"requires": {
"ms": "^2.1.1"
}
}
}
},
"for-in": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
@@ -1955,8 +1982,7 @@
"is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
"dev": true
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
},
"is-ci": {
"version": "1.2.1",

1
package.json
View File

@@ -21,6 +21,7 @@
"node": "11.12.x"
},
"dependencies": {
"axios": "^0.18.0",
"csv-parser": "^2.3.0",
"dotenv": "^8.0.0",
"express": "^4.17.0",

162
services/doorLock.js
View File

@@ -14,84 +14,110 @@ const {
csvParserErrors,
} = require('../constants/constants');
const { fetchAllMembers, findMember } = require('../services/officeRnD/members');
const parseDoorLockDataFile = (file) => {
return new Promise ((resolve, reject) => {
const results = [];
const errors = [];
const unknownMembers = [];
let isValidFile = true;
fs.createReadStream(file.path)
.pipe(csv({
mapHeaders: ({ header, index }) => header.trim().toLowerCase(),
mapValues: ({ header, index, value }) => value.trim()
}))
.on('headers', (headers) => {
headers.forEach((header) => {
if (!VALID_CSV_HEADERS.includes(header.trim())){
isValidFile = false;
console.log('INVALID HEADER');
errors.push({
error: csvParserErrors.UNKNOWN_COLUMN,
details: header,
file: file.name,
fetchAllMembers()
.then(() => {
fs.createReadStream(file.path)
.pipe(csv({
mapHeaders: ({ header, index }) => header.trim().toLowerCase(),
mapValues: ({ header, index, value }) => value.trim()
}))
.on('headers', (headers) => {
headers.forEach((header) => {
if (!VALID_CSV_HEADERS.includes(header.trim())){
isValidFile = false;
console.log('INVALID HEADER');
errors.push({
error: csvParserErrors.UNKNOWN_COLUMN,
details: header,
file: file.name,
});
}
});
}
});
})
.on('data', (data) => {
if (!isValidFile) {
return;
}
const eventType = data.event.trim();
if ([USER_ENTRY_EVENT, ENABLE_PASSAGE_MODE, DISABLE_PASSAGE_MODE].includes(eventType)){
results.push(data);
}
})
.on('end', () => {
const parsedData = [];
let i = 0;
while (i < results.length){
//Verify pair
//First entry should be user entry and second should be enable / disable passage
const firstEntry = results[i];
const secondEntry = results[i+1];
if (firstEntry && (firstEntry.event === USER_ENTRY_EVENT)){
if (secondEntry && (secondEntry.event === ENABLE_PASSAGE_MODE || secondEntry.event === DISABLE_PASSAGE_MODE)){
const event = (secondEntry.event === ENABLE_PASSAGE_MODE) ? USER_UNLOCKED_DOOR : USER_LOCKED_DOOR;
const entryData = {
memberName: firstEntry.name,
memberNumber: firstEntry['user no'],
date: firstEntry.date,
time: firstEntry.time,
event,
};
parsedData.push(entryData);
i+=2;
} else {
errors.push({
error: csvParserErrors.INVALID_ENTRY_EXPECTED_PASSAGE_MODE,
details: secondEntry || 'Last row in file',
file: file.name,
});
i+=1;
})
.on('data', (data) => {
if (!isValidFile) {
return;
}
} else {
errors.push({
error: csvParserErrors.INVALID_ENTRY_EXPECTED_USER,
details: firstEntry,
file: file.name,
const eventType = data.event.trim();
if ([USER_ENTRY_EVENT, ENABLE_PASSAGE_MODE, DISABLE_PASSAGE_MODE].includes(eventType)){
results.push(data);
}
})
.on('end', () => {
const parsedData = [];
let i = 0;
while (i < results.length){
//Verify pair
//First entry should be user entry and second should be enable / disable passage
const firstEntry = results[i];
const secondEntry = results[i+1];
if (firstEntry && (firstEntry.event === USER_ENTRY_EVENT)){
if (secondEntry && (secondEntry.event === ENABLE_PASSAGE_MODE || secondEntry.event === DISABLE_PASSAGE_MODE)){
const event = (secondEntry.event === ENABLE_PASSAGE_MODE) ? USER_UNLOCKED_DOOR : USER_LOCKED_DOOR;
const memberObject = findMember(firstEntry.name);
//Verify that member is registered in OfficeRnD system
if (memberObject){
const entryData = {
memberName: firstEntry.name,
memberNumber: firstEntry['user no'],
memberId: memberObject.memberId,
date: firstEntry.date,
time: firstEntry.time,
event,
};
parsedData.push(entryData);
} else {
//Check if member is already labeled as unknown
const unknownMember = unknownMembers.find((member) => member.details === firstEntry.name);
if (!unknownMember){
unknownMembers.push({
error: csvParserErrors.UNKNOWN_MEMBER,
details: firstEntry.name,
file: file.name,
});
}
}
i+=2;
} else {
errors.push({
error: csvParserErrors.INVALID_ENTRY_EXPECTED_PASSAGE_MODE,
details: secondEntry || 'Last row in file',
file: file.name,
});
i+=1;
}
} else {
errors.push({
error: csvParserErrors.INVALID_ENTRY_EXPECTED_USER,
details: firstEntry,
file: file.name,
});
i+=1;
}
}
resolve({
parsedData,
unknownMembers,
errors
});
i+=1;
}
}
resolve({
parsedData,
errors
});
});
})
.catch((error) => {
reject(error);
});
});
};

33
services/officeRnD/members.js Normal file
View File

@@ -0,0 +1,33 @@
'use strict';
const { API } = require('../../helpers/api');
const membersList = [];
const fetchAllMembers = () => {
return new Promise((resolve, reject) => {
API.get('/members')
.then((result) => {
const members = result.data || [];
members.forEach((member) => {
membersList.push({
name: member.name,
memberId: member['_id'],
});
});
resolve();
})
.catch((error) => {
reject(error);
});
});
};
const findMember = (memberName) => {
return membersList.find((member) => member.name === memberName);
};
module.exports = {
fetchAllMembers,
findMember,
};