diff --git a/client/build/design.zip b/client/build/design.zip new file mode 100644 index 0000000..ec3db11 Binary files /dev/null and b/client/build/design.zip differ diff --git a/client/config/development/style.js b/client/config/development/style.js index 6d67a8e..eec04d4 100644 --- a/client/config/development/style.js +++ b/client/config/development/style.js @@ -1,5 +1,6 @@ // Vendor Stylesheets require('bootstrap/dist/css/bootstrap.min.css'); +require('c3/c3.min.css') require(__dirname + '/../../d3/chart.scss'); // Component Stylesheets diff --git a/client/dashboard/power/graph/graph.component.js b/client/dashboard/power/graph/graph.component.js index 83fde0a..0da9a0b 100644 --- a/client/dashboard/power/graph/graph.component.js +++ b/client/dashboard/power/graph/graph.component.js @@ -2,7 +2,7 @@ import React from 'react'; import Templates from 'config/templates'; import House from './../../../models/house'; -import SplineStackChart from './../../../d3/line/spline_stack'; +import c3 from 'c3'; class GraphComponent extends React.Component { @@ -28,53 +28,38 @@ class GraphComponent extends React.Component { updateGraph(){ var power_graph = this, - house = power_graph.house; + house = power_graph.house, + data = { + x: 'x', + json: house.power_data, + keys: { + x: 'time_to_date', + value: ['net_consumption', 'production'] + }, + type: 'area-spline', + groups: [['net_consumption', 'production']], + names: { + net_consumption: 'Net Consumption', + production: 'Production' + } + }; if (power_graph.graph === undefined){ - power_graph.graph = new SplineStackChart({ - container: '#power_graph', - outer_width: 800, - outer_height: 200, - color: '#0404B4', - time_series: true, - domain_attr: 'x', - range_attr: 'y', - include_dots: true, - titleizeDatum: (series, d)=>{ - return series.title + '
' + Math.round(d.y) + ' W
' + house.formatDate(d.power_graph_datum.data.time, 'MMM D [at] HH:mm'); + power_graph.chart = c3.generate({ + bindto: '#power_graph', + data: data, + axis: { + x: { + type: 'timeseries', + tick: { format: d3.time.format('%d %B %y') } + } } }); - jQuery('#power_graph').tooltip({ - selector: 'circle', - container: 'body', - html: true, - title: function(){ - return this.__data__.title; - } + } else { + power_graph.chart.load({ + unload: true, + data: data }); } - var net_power_graph = { - title: 'Net Power Consumption', - values: house.power_data.map((power_graph_datum)=>{ - return { - power_graph_datum: power_graph_datum, - x: power_graph_datum.time_to_date, - y: Math.max(0, power_graph_datum.data.consumption - power_graph_datum.data.production) } - }) - }, - savings = { - title: 'Power Production', - values: house.power_data.map((power_graph_datum)=>{ - return { - power_graph_datum: power_graph_datum, - x: power_graph_datum.time_to_date, - y: power_graph_datum.data.production } - }) - }; - power_graph.graph.drawData({ - title: power_graph.graph_title, - css_class: '', - series: [net_power_graph, savings] - }); } render() { diff --git a/client/models/power_datum.js b/client/models/power_datum.js index 057e295..14aa4e8 100644 --- a/client/models/power_datum.js +++ b/client/models/power_datum.js @@ -24,16 +24,20 @@ class PowerDatum { return house.toDate(power_datum.data.time); } + get net_consumption(){ + return Math.max(0, Math.round(this.data.consumption - this.data.production)); + } + get time_to_s(){ var power_datum = this, moment_tz = moment.tz(power_datum.data.time * 1000, power_datum.house.data.timezone); return moment_tz.format('YYYY-MM-DD HH:mm'); } - get consumption_to_s(){ + get consumption(){ var power_datum = this; return Math.round(power_datum.data.consumption); } - get production_to_s(){ + get production(){ var power_datum = this; return Math.round(power_datum.data.production); } diff --git a/gulpfile.babel.js b/gulpfile.babel.js index 4d09a73..d1100e3 100644 --- a/gulpfile.babel.js +++ b/gulpfile.babel.js @@ -2,11 +2,12 @@ import gulp from 'gulp'; import yargs from 'yargs'; import webpack from 'webpack'; import gutil from 'gulp-util'; -import gulpCopy from 'gulp-copy' +import gulpCopy from 'gulp-copy'; import fs from 'fs'; import FsHelper from './server/lib/fs_helper'; import ComponentMapWriter from './server/lib/tasks/component_map_writer'; +import DesignDataGenerator from './server/lib/tasks/design_data_generator'; import BuildDashboardAssets from './server/lib/tasks/build_dashboard_assets'; import DB from './server/config/database'; import {PowerDataSeed, HouseSeed} from './server/lib/tasks/seed_data'; @@ -17,6 +18,19 @@ gulp.task('generate_power_csv', function(done){ }); }); +gulp.task('generate_design_data', function(){ + var exec = require('child_process').exec, + house_ids = yargs.argv.house_ids.split(','), + start_date = parseInt(yargs.argv.start_date), + end_date = parseInt(yargs.argv.end_date), + data_generator = new DesignDataGenerator(house_ids, [start_date, end_date]); + return DB.sync() + .then(()=>{ + return data_generator.exec(); + }); +}); + + gulp.task('save_power_csv', function(done){ DB.sync().then(()=>{ PowerDataSeed.saveCsv(yargs.argv, done); @@ -53,9 +67,7 @@ gulp.task('build', function(done) { var path = 'client/build/design/dashboard'; // copy all react templates and their styles sheets into build/design/dashboard. FsHelper.rmdirAsync(path, ()=>{ - console.log('path removed') fs.mkdir(path, ()=>{ - console.log('path recreated') gulp.src([ `client/app.scss` ]).pipe(gulpCopy(path, {prefix: 1})); @@ -65,7 +77,6 @@ gulp.task('build', function(done) { var files_to_copy = files.filter((file)=>{ return /\.(rt|scss)$/.test(file) }); - console.log(files_to_copy) gulp.src(files_to_copy) .pipe(gulpCopy(path, {prefix: 2})); done() diff --git a/package.json b/package.json index 1709b1c..a0f5c82 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "jquery": "2.2.0", "bootstrap": "3.3.6", "d3": "3.5.12", + "c3": "0.4.11-rc4", "raw-loader": "0.5.1", "sass-loader": "3.1.2", "style-loader": "^0.12.3", @@ -41,6 +42,7 @@ "react-templates-loader": "0.4.0", "node-sass": "3.4.2", "moment-timezone": "0.5.0", + "cheerio": "0.20.0", "yargs": "3.32.0", "extend": "3.0.0", "through2": "2.0.1", diff --git a/server/controllers/houses_controller.js b/server/controllers/houses_controller.js index 3fa4d8e..f6a30e7 100644 --- a/server/controllers/houses_controller.js +++ b/server/controllers/houses_controller.js @@ -6,7 +6,7 @@ class HousesController { static index(req, res){ var params = {}; - if (req.query.ids) query.id = ids; + if (req.query.ids) params.id = ids; DB.House.findAll({where: params}).then((houses)=>{ res.json({data: houses.map((house)=>{ return house.dataValues; })}); }); diff --git a/server/lib/data_helper.js b/server/lib/data_helper.js new file mode 100644 index 0000000..2d0a26b --- /dev/null +++ b/server/lib/data_helper.js @@ -0,0 +1,27 @@ +import cheerio from 'cheerio'; +import fs from 'fs'; +import moment from 'moment'; + +class DataHelper { + + static baseIrradiance(){ + return new Promise((fnResolve, fnReject)=>{ + fs.readFile(__dirname + '/../data/irradiance.html', (err, data)=>{ + if (err) return fnReject(err); + var $ = cheerio.load(data), + base_irradiance = {}; + $('tbody tr').each((i, elem)=>{ + if (i === 0) return true; + var cells = $(elem).find('td'), + date_s = moment($(cells[1]).text(), 'YYYY-MM-DD').format('MM-DD'), + irradiance = parseFloat($(cells[2]).text()); + base_irradiance[date_s] = irradiance; + }); + fnResolve(base_irradiance); + }); + }); + } + +} + +export default DataHelper; diff --git a/server/lib/tasks/design_data_generator.js b/server/lib/tasks/design_data_generator.js new file mode 100644 index 0000000..54b4026 --- /dev/null +++ b/server/lib/tasks/design_data_generator.js @@ -0,0 +1,113 @@ +import fs from 'fs'; + +import FsHelper from './../fs_helper'; +import DB from './../../config/database'; + +const DESIGN_DATA_PATH = __dirname + '/../../../client/build/design/data'; + +class DesignDataGenerator { + + constructor(house_ids, dates){ + var generator = this; + generator.house_ids = house_ids; + generator.dates = dates; + } + + exec(){ + var generator = this; + console.log('Clearing design data directory...') + return generator.clearDirectory() + .then(()=>{ + console.log('Writing house index response...') + return generator.writeHouseIndex(); + }) + .then(()=>{ + console.log('Writing house energy and power data...'); + return generator.writeHouseData(); + }) + .then(()=>{ + console.log('Done!') + }); + } + + clearDirectory(){ + console.log('DesignDataGenerator#clearDirectory') + return new Promise((fnResolve1, fnReject1)=>{ + // remove directory & contents + FsHelper.rmdirAsync(DESIGN_DATA_PATH, ()=>{ + // recreate it. + fs.mkdir(DESIGN_DATA_PATH, ()=>{ + // create subdirectories + Promise.all([ + new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/energy_data', fnResolve2); }), + new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/power_data', fnResolve2); }) + ]).then(fnResolve1); + }); + }); + }); + } + + writeHouseData(){ + var generator = this; + return DB.House.findAll({where: {id: generator.house_ids}}) + .then((houses)=>{ + var promises = []; + // for all houses, write energy and power responses. + houses.forEach((house)=>{ + promises.push(new Promise((fnResolve, fnReject)=>{ + DesignDataGenerator.energyIndex({house_id: house.id, dates: [generator.dates]}) + .then((json)=>{ + fs.writeFile(DESIGN_DATA_PATH + `/energy_data/${house.id}.json`, json, fnResolve); + }); + })); + promises.push(new Promise((fnResolve, fnReject)=>{ + DesignDataGenerator.powerIndex({house_id: house.id, dates: [generator.dates]}) + .then((json)=>{ + fs.writeFile(DESIGN_DATA_PATH + `/power_data/${house.id}.json`, json, fnResolve); + }); + })); + }); + return Promise.all(promises); + }); + } + + writeHouseIndex(){ + var generator = this; + return new Promise((fnResolve, fnReject)=>{ + DesignDataGenerator.housesIndex({id: generator.house_ids, dates: generator.dates}) + .then((json)=>{ + fs.writeFile(DESIGN_DATA_PATH + '/houses.json', json, fnResolve); + }); + }); + } + + static housesIndex(opts){ + return DB.House.findAll({where: {id: opts.id}}) + .then((houses_data)=>{ + if (opts.dates){ + houses_data.forEach((house_datum)=>{ + house_datum.data_from = opts.dates[0]; + house_datum.data_until = opts.dates[1]; + }); + } + return JSON.stringify(houses_data, null, 2); + }); + } + + static powerIndex(opts){ + return DB.PowerDatum.exposeForHouseAtDates(opts.house_id, opts.dates) + .then((power_data)=>{ + return JSON.stringify({data: power_data}); + }); + } + + static energyIndex(opts){ + return DB.EnergyDatum.exposeForHouseAtDates(opts.house_id, opts.dates) + .then((energy_data)=>{ + return JSON.stringify({data: energy_data}); + }); + } + +} + +export default DesignDataGenerator; diff --git a/server/lib/tasks/seed_data.js b/server/lib/tasks/seed_data.js index 1732dd0..6b7e0ac 100644 --- a/server/lib/tasks/seed_data.js +++ b/server/lib/tasks/seed_data.js @@ -5,20 +5,22 @@ import fs from 'fs'; import MathUtils from "./../../../shared/utils/math" import DB from './../../config/database'; -const DATA_PATH = __dirname + '/../../../shared/data/' +const DATA_PATH = __dirname + '/../../data/' export class PowerDataSeed { static saveCsv(opts, done){ opts = extend({ - path: DATA_PATH + "power_data.csv" + path: "power_data.csv", + house_id: null }, opts || {}); + opts.path = DATA_PATH + opts.path; var stream = fs.createReadStream(opts.path), - csvStream = csv.fromStream(stream, {headers: ['house_id', 'time', 'consumption', 'production']}), + csvStream = csv.fromStream(stream, {headers: ['time', 'consumption', 'production']}), rows = []; csvStream.on("data", function(data){ - data.time = data.time; + data.house_id = opts.house_id rows.push(data); if (rows.length % 100 === 0){ DB.PowerDatum.bulkCreate(rows, {validate: true}).catch((error)=>{ @@ -31,13 +33,8 @@ export class PowerDataSeed { csvStream.on("end", function(){ console.log("all rows parsed") DB.PowerDatum.bulkCreate(rows, {validate: true}).then(()=>{ - return DB.House.findAll().then((houses)=>{ - var promises = []; - for (var house of houses){ - var p = house.aggregatePowerToEnergyData(); - promises.push(p); - } - return Promise.all(promises); + return DB.House.findOne({where: {id: opts.house_id}}).then((house)=>{ + return house.aggregatePowerToEnergyData(); }); }).then(()=>{ console.log("DONE!") @@ -51,32 +48,31 @@ export class PowerDataSeed { end_date: moment().unix(), interval: 900, // every 15 minutes (in s) average: 1400, // Wh - path: DATA_PATH + "power_data.csv" + path: "power_data.csv" }, opts || {}); -console.log(opts.start_date, opts.end_date) + opts.path = DATA_PATH + opts.path; + opts.production_multiplier = parseFloat(opts.production_multiplier); + var row_date = opts.start_date, csvStream = csv.format({headers: true}), - writableStream = fs.createWriteStream(opts.path), - house_ids = opts.house_ids.split(",") + writableStream = fs.createWriteStream(opts.path); - DB.House.findAll({where: {id: house_ids}}).then((houses)=>{ + csvStream.pipe(writableStream); + writableStream.on("finish", ()=>{ + console.log("DONE!") + done(); + }); - csvStream.pipe(writableStream); - writableStream.on("finish", ()=>{ - console.log("DONE!") - done(); - }); - - while (row_date <= opts.end_date){ - for (var house of houses){ + DB.House.findOne({where: {id: opts.house_id}}) + .then((house)=>{ + while (row_date <= opts.end_date){ var consumption = MathUtils.normal(opts.average), production = MathUtils.normal(opts.average) * house.productionMultiplier(row_date * 1000); - csvStream.write([house.id, row_date, consumption, production]); + csvStream.write([row_date, consumption, production]); + row_date += opts.interval; } - row_date += opts.interval; - } - csvStream.end(); - }); + csvStream.end(); + }) } } @@ -86,7 +82,7 @@ export class HouseSeed { path: DATA_PATH + "houses.csv" }, opts || {}); var stream = fs.createReadStream(opts.path), - csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'timezone']}), + csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'production_multiplier', 'timezone']}), rows = []; csvStream.on("data", function(data){ diff --git a/server/models/energy_datum.js b/server/models/energy_datum.js index 147985f..fc62db7 100644 --- a/server/models/energy_datum.js +++ b/server/models/energy_datum.js @@ -17,6 +17,7 @@ var EnergyDatum = DB.sequelize.define(NAME, { day: { type: DB.Sequelize.INTEGER, }, + irradiance: DB.Sequelize.FLOAT, production: DB.Sequelize.FLOAT, consumption: DB.Sequelize.FLOAT }, { @@ -34,8 +35,6 @@ var EnergyDatum = DB.sequelize.define(NAME, { exposeForHouseAtDates: (house_id, dates)=>{ var params = {house_id: house_id}; extend(params, ApiHelper.datesParamToSequelize(dates, 'day')); - console.log('EnergyDatum#exposeForHouseAtDates') - console.log(params, dates) return EnergyDatum.findAll({ where: params, attributes: ['id', 'production', 'consumption', 'day'] diff --git a/server/models/house.js b/server/models/house.js index 2700180..6d16bbd 100644 --- a/server/models/house.js +++ b/server/models/house.js @@ -1,5 +1,6 @@ import moment from 'moment-timezone'; import DB from "./../config/database"; +import DataHelper from './../lib/data_helper'; const NAME = 'House'; @@ -15,6 +16,7 @@ var House = DB.sequelize.define(NAME, { }, timezone: DB.Sequelize.STRING, name: DB.Sequelize.STRING, + production_multiplier: DB.Sequelize.FLOAT, data_until: { type: DB.Sequelize.INTEGER, }, @@ -33,37 +35,53 @@ var House = DB.sequelize.define(NAME, { if (minute > 420 && minute < 1140){ multiplier = 1 - Math.abs(780 - minute) / 360; } - return multiplier; + return multiplier * house.production_multiplier; }, unixToLocalDay: function(unix){ var house = this; return moment.tz(unix * 1000, house.timezone).startOf('day').unix(); }, - aggregatePowerToEnergyData: function(){ + dayToMonthDayString: function(unix){ var house = this; + return moment.tz(unix * 1000, house.timezone).format('MM-DD'); + }, + aggregatePowerToEnergyData: function(){ + var house = this, + base_irradiance; return DB.EnergyDatum.destroy({where: {house_id: house.id}}) .then(()=>{ + return DataHelper.baseIrradiance(); + }) + .then((data)=>{ + base_irradiance = data; return DB.PowerDatum.count({where: {house_id: house.id}}) }) .then((count)=>{ var limit = 0, energy_data = new Map(), promises = []; + console.log('House#aggregatePowerToEnergyData') while (limit < count){ let complete = DB.PowerDatum.findAll({where: {house_id: house.id}, limit: 1000, offset: limit, order: 'id ASC'}) - .then((power_data)=>{ - power_data.forEach((power_datum)=>{ - var day = house.unixToLocalDay(power_datum.time), - energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id}; - energy_datum.production += power_datum.production / 1000; // convert Wh to kWh - energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh - energy_data.set(day, energy_datum); - }); - }); + .then((power_data)=>{ + power_data.forEach((power_datum)=>{ + var day = house.unixToLocalDay(power_datum.time), + energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id}; + energy_datum.production += power_datum.production / 1000; // convert Wh to kWh + energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh + energy_data.set(day, energy_datum); + }); + }); promises.push(complete); limit += 1000; } return Promise.all(promises).then(()=>{ + Array.from(energy_data.values()).forEach((energy_datum)=>{ + let day_multiplier = 1 + Math.random() * 0.10 - 0.05, + base_day_irradiance = base_irradiance[house.dayToMonthDayString(energy_datum.day)] || 105, + irradiance = base_day_irradiance * house.production_multiplier * day_multiplier; + energy_datum.irradiance = irradiance; + }) return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true}); }); })