senaduka/old-spike
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use c3 for stacked spline

Browse Source
This commit is contained in:
Eric Hulburd
2016-03-08 15:21:16 -06:00
parent bfdebcf8f9
commit 2dd9389694
12 changed files with 249 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
client/build/design.zip Normal file
View File

Binary file not shown.

1
client/config/development/style.js
View File

@@ -1,5 +1,6 @@
// Vendor Stylesheets
require('bootstrap/dist/css/bootstrap.min.css');
require('c3/c3.min.css')
require(__dirname + '/../../d3/chart.scss');
// Component Stylesheets

71
client/dashboard/power/graph/graph.component.js
View File

@@ -2,7 +2,7 @@ import React from 'react';
import Templates from 'config/templates';
import House from './../../../models/house';
import SplineStackChart from './../../../d3/line/spline_stack';
import c3 from 'c3';
class GraphComponent extends React.Component {
@@ -28,53 +28,38 @@ class GraphComponent extends React.Component {
updateGraph(){
var power_graph = this,
house = power_graph.house;
house = power_graph.house,
data = {
x: 'x',
json: house.power_data,
keys: {
x: 'time_to_date',
value: ['net_consumption', 'production']
},
type: 'area-spline',
groups: [['net_consumption', 'production']],
names: {
net_consumption: 'Net Consumption',
production: 'Production'
}
};
if (power_graph.graph === undefined){
power_graph.graph = new SplineStackChart({
container: '#power_graph',
outer_width: 800,
outer_height: 200,
color: '#0404B4',
time_series: true,
domain_attr: 'x',
range_attr: 'y',
include_dots: true,
titleizeDatum: (series, d)=>{
return series.title + '<br/>' + Math.round(d.y) + ' W<br/>' + house.formatDate(d.power_graph_datum.data.time, 'MMM D [at] HH:mm');
power_graph.chart = c3.generate({
bindto: '#power_graph',
data: data,
axis: {
x: {
type: 'timeseries',
tick: { format: d3.time.format('%d %B %y') }
}
}
});
jQuery('#power_graph').tooltip({
selector: 'circle',
container: 'body',
html: true,
title: function(){
return this.__data__.title;
}
} else {
power_graph.chart.load({
unload: true,
data: data
});
}
var net_power_graph = {
title: 'Net Power Consumption',
values: house.power_data.map((power_graph_datum)=>{
return {
power_graph_datum: power_graph_datum,
x: power_graph_datum.time_to_date,
y: Math.max(0, power_graph_datum.data.consumption - power_graph_datum.data.production) }
})
},
savings = {
title: 'Power Production',
values: house.power_data.map((power_graph_datum)=>{
return {
power_graph_datum: power_graph_datum,
x: power_graph_datum.time_to_date,
y: power_graph_datum.data.production }
})
};
power_graph.graph.drawData({
title: power_graph.graph_title,
css_class: '',
series: [net_power_graph, savings]
});
}
render() {

8
client/models/power_datum.js
View File

@@ -24,16 +24,20 @@ class PowerDatum {
return house.toDate(power_datum.data.time);
}
get net_consumption(){
return Math.max(0, Math.round(this.data.consumption - this.data.production));
}
get time_to_s(){
var power_datum = this,
moment_tz = moment.tz(power_datum.data.time * 1000, power_datum.house.data.timezone);
return moment_tz.format('YYYY-MM-DD HH:mm');
}
get consumption_to_s(){
get consumption(){
var power_datum = this;
return Math.round(power_datum.data.consumption);
}
get production_to_s(){
get production(){
var power_datum = this;
return Math.round(power_datum.data.production);
}

19
gulpfile.babel.js
View File

@@ -2,11 +2,12 @@ import gulp from 'gulp';
import yargs from 'yargs';
import webpack from 'webpack';
import gutil from 'gulp-util';
import gulpCopy from 'gulp-copy'
import gulpCopy from 'gulp-copy';
import fs from 'fs';
import FsHelper from './server/lib/fs_helper';
import ComponentMapWriter from './server/lib/tasks/component_map_writer';
import DesignDataGenerator from './server/lib/tasks/design_data_generator';
import BuildDashboardAssets from './server/lib/tasks/build_dashboard_assets';
import DB from './server/config/database';
import {PowerDataSeed, HouseSeed} from './server/lib/tasks/seed_data';
@@ -17,6 +18,19 @@ gulp.task('generate_power_csv', function(done){
});
});
gulp.task('generate_design_data', function(){
var exec = require('child_process').exec,
house_ids = yargs.argv.house_ids.split(','),
start_date = parseInt(yargs.argv.start_date),
end_date = parseInt(yargs.argv.end_date),
data_generator = new DesignDataGenerator(house_ids, [start_date, end_date]);
return DB.sync()
.then(()=>{
return data_generator.exec();
});
});
gulp.task('save_power_csv', function(done){
DB.sync().then(()=>{
PowerDataSeed.saveCsv(yargs.argv, done);
@@ -53,9 +67,7 @@ gulp.task('build', function(done) {
var path = 'client/build/design/dashboard';
// copy all react templates and their styles sheets into build/design/dashboard.
FsHelper.rmdirAsync(path, ()=>{
console.log('path removed')
fs.mkdir(path, ()=>{
console.log('path recreated')
gulp.src([
`client/app.scss`
]).pipe(gulpCopy(path, {prefix: 1}));
@@ -65,7 +77,6 @@ gulp.task('build', function(done) {
var files_to_copy = files.filter((file)=>{
return /\.(rt|scss)$/.test(file)
});
console.log(files_to_copy)
gulp.src(files_to_copy)
.pipe(gulpCopy(path, {prefix: 2}));
done()

2
package.json
View File

@@ -34,6 +34,7 @@
"jquery": "2.2.0",
"bootstrap": "3.3.6",
"d3": "3.5.12",
"c3": "0.4.11-rc4",
"raw-loader": "0.5.1",
"sass-loader": "3.1.2",
"style-loader": "^0.12.3",
@@ -41,6 +42,7 @@
"react-templates-loader": "0.4.0",
"node-sass": "3.4.2",
"moment-timezone": "0.5.0",
"cheerio": "0.20.0",
"yargs": "3.32.0",
"extend": "3.0.0",
"through2": "2.0.1",

2
server/controllers/houses_controller.js
View File

@@ -6,7 +6,7 @@ class HousesController {
static index(req, res){
var params = {};
if (req.query.ids) query.id = ids;
if (req.query.ids) params.id = ids;
DB.House.findAll({where: params}).then((houses)=>{
res.json({data: houses.map((house)=>{ return house.dataValues; })});
});

27
server/lib/data_helper.js Normal file
View File

@@ -0,0 +1,27 @@
import cheerio from 'cheerio';
import fs from 'fs';
import moment from 'moment';
class DataHelper {
static baseIrradiance(){
return new Promise((fnResolve, fnReject)=>{
fs.readFile(__dirname + '/../data/irradiance.html', (err, data)=>{
if (err) return fnReject(err);
var $ = cheerio.load(data),
base_irradiance = {};
$('tbody tr').each((i, elem)=>{
if (i === 0) return true;
var cells = $(elem).find('td'),
date_s = moment($(cells[1]).text(), 'YYYY-MM-DD').format('MM-DD'),
irradiance = parseFloat($(cells[2]).text());
base_irradiance[date_s] = irradiance;
});
fnResolve(base_irradiance);
});
});
}
}
export default DataHelper;

113
server/lib/tasks/design_data_generator.js Normal file
View File

@@ -0,0 +1,113 @@
import fs from 'fs';
import FsHelper from './../fs_helper';
import DB from './../../config/database';
const DESIGN_DATA_PATH = __dirname + '/../../../client/build/design/data';
class DesignDataGenerator {
constructor(house_ids, dates){
var generator = this;
generator.house_ids = house_ids;
generator.dates = dates;
}
exec(){
var generator = this;
console.log('Clearing design data directory...')
return generator.clearDirectory()
.then(()=>{
console.log('Writing house index response...')
return generator.writeHouseIndex();
})
.then(()=>{
console.log('Writing house energy and power data...');
return generator.writeHouseData();
})
.then(()=>{
console.log('Done!')
});
}
clearDirectory(){
console.log('DesignDataGenerator#clearDirectory')
return new Promise((fnResolve1, fnReject1)=>{
// remove directory & contents
FsHelper.rmdirAsync(DESIGN_DATA_PATH, ()=>{
// recreate it.
fs.mkdir(DESIGN_DATA_PATH, ()=>{
// create subdirectories
Promise.all([
new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/energy_data', fnResolve2); }),
new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/power_data', fnResolve2); })
]).then(fnResolve1);
});
});
});
}
writeHouseData(){
var generator = this;
return DB.House.findAll({where: {id: generator.house_ids}})
.then((houses)=>{
var promises = [];
// for all houses, write energy and power responses.
houses.forEach((house)=>{
promises.push(new Promise((fnResolve, fnReject)=>{
DesignDataGenerator.energyIndex({house_id: house.id, dates: [generator.dates]})
.then((json)=>{
fs.writeFile(DESIGN_DATA_PATH + `/energy_data/${house.id}.json`, json, fnResolve);
});
}));
promises.push(new Promise((fnResolve, fnReject)=>{
DesignDataGenerator.powerIndex({house_id: house.id, dates: [generator.dates]})
.then((json)=>{
fs.writeFile(DESIGN_DATA_PATH + `/power_data/${house.id}.json`, json, fnResolve);
});
}));
});
return Promise.all(promises);
});
}
writeHouseIndex(){
var generator = this;
return new Promise((fnResolve, fnReject)=>{
DesignDataGenerator.housesIndex({id: generator.house_ids, dates: generator.dates})
.then((json)=>{
fs.writeFile(DESIGN_DATA_PATH + '/houses.json', json, fnResolve);
});
});
}
static housesIndex(opts){
return DB.House.findAll({where: {id: opts.id}})
.then((houses_data)=>{
if (opts.dates){
houses_data.forEach((house_datum)=>{
house_datum.data_from = opts.dates[0];
house_datum.data_until = opts.dates[1];
});
}
return JSON.stringify(houses_data, null, 2);
});
}
static powerIndex(opts){
return DB.PowerDatum.exposeForHouseAtDates(opts.house_id, opts.dates)
.then((power_data)=>{
return JSON.stringify({data: power_data});
});
}
static energyIndex(opts){
return DB.EnergyDatum.exposeForHouseAtDates(opts.house_id, opts.dates)
.then((energy_data)=>{
return JSON.stringify({data: energy_data});
});
}
}
export default DesignDataGenerator;

56
server/lib/tasks/seed_data.js
View File

@@ -5,20 +5,22 @@ import fs from 'fs';
import MathUtils from "./../../../shared/utils/math"
import DB from './../../config/database';
const DATA_PATH = __dirname + '/../../../shared/data/'
const DATA_PATH = __dirname + '/../../data/'
export class PowerDataSeed {
static saveCsv(opts, done){
opts = extend({
path: DATA_PATH + "power_data.csv"
path: "power_data.csv",
house_id: null
}, opts || {});
opts.path = DATA_PATH + opts.path;
var stream = fs.createReadStream(opts.path),
csvStream = csv.fromStream(stream, {headers: ['house_id', 'time', 'consumption', 'production']}),
csvStream = csv.fromStream(stream, {headers: ['time', 'consumption', 'production']}),
rows = [];
csvStream.on("data", function(data){
data.time = data.time;
data.house_id = opts.house_id
rows.push(data);
if (rows.length % 100 === 0){
DB.PowerDatum.bulkCreate(rows, {validate: true}).catch((error)=>{
@@ -31,13 +33,8 @@ export class PowerDataSeed {
csvStream.on("end", function(){
console.log("all rows parsed")
DB.PowerDatum.bulkCreate(rows, {validate: true}).then(()=>{
return DB.House.findAll().then((houses)=>{
var promises = [];
for (var house of houses){
var p = house.aggregatePowerToEnergyData();
promises.push(p);
}
return Promise.all(promises);
return DB.House.findOne({where: {id: opts.house_id}}).then((house)=>{
return house.aggregatePowerToEnergyData();
});
}).then(()=>{
console.log("DONE!")
@@ -51,32 +48,31 @@ export class PowerDataSeed {
end_date: moment().unix(),
interval: 900, // every 15 minutes (in s)
average: 1400, // Wh
path: DATA_PATH + "power_data.csv"
path: "power_data.csv"
}, opts || {});
console.log(opts.start_date, opts.end_date)
opts.path = DATA_PATH + opts.path;
opts.production_multiplier = parseFloat(opts.production_multiplier);
var row_date = opts.start_date,
csvStream = csv.format({headers: true}),
writableStream = fs.createWriteStream(opts.path),
house_ids = opts.house_ids.split(",")
writableStream = fs.createWriteStream(opts.path);
DB.House.findAll({where: {id: house_ids}}).then((houses)=>{
csvStream.pipe(writableStream);
writableStream.on("finish", ()=>{
console.log("DONE!")
done();
});
csvStream.pipe(writableStream);
writableStream.on("finish", ()=>{
console.log("DONE!")
done();
});
while (row_date <= opts.end_date){
for (var house of houses){
DB.House.findOne({where: {id: opts.house_id}})
.then((house)=>{
while (row_date <= opts.end_date){
var consumption = MathUtils.normal(opts.average),
production = MathUtils.normal(opts.average) * house.productionMultiplier(row_date * 1000);
csvStream.write([house.id, row_date, consumption, production]);
csvStream.write([row_date, consumption, production]);
row_date += opts.interval;
}
row_date += opts.interval;
}
csvStream.end();
});
csvStream.end();
})
}
}
@@ -86,7 +82,7 @@ export class HouseSeed {
path: DATA_PATH + "houses.csv"
}, opts || {});
var stream = fs.createReadStream(opts.path),
csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'timezone']}),
csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'production_multiplier', 'timezone']}),
rows = [];
csvStream.on("data", function(data){

3
server/models/energy_datum.js
View File

@@ -17,6 +17,7 @@ var EnergyDatum = DB.sequelize.define(NAME, {
day: {
type: DB.Sequelize.INTEGER,
},
irradiance: DB.Sequelize.FLOAT,
production: DB.Sequelize.FLOAT,
consumption: DB.Sequelize.FLOAT
}, {
@@ -34,8 +35,6 @@ var EnergyDatum = DB.sequelize.define(NAME, {
exposeForHouseAtDates: (house_id, dates)=>{
var params = {house_id: house_id};
extend(params, ApiHelper.datesParamToSequelize(dates, 'day'));
console.log('EnergyDatum#exposeForHouseAtDates')
console.log(params, dates)
return EnergyDatum.findAll({
where: params,
attributes: ['id', 'production', 'consumption', 'day']

40
server/models/house.js
View File

@@ -1,5 +1,6 @@
import moment from 'moment-timezone';
import DB from "./../config/database";
import DataHelper from './../lib/data_helper';
const NAME = 'House';
@@ -15,6 +16,7 @@ var House = DB.sequelize.define(NAME, {
},
timezone: DB.Sequelize.STRING,
name: DB.Sequelize.STRING,
production_multiplier: DB.Sequelize.FLOAT,
data_until: {
type: DB.Sequelize.INTEGER,
},
@@ -33,37 +35,53 @@ var House = DB.sequelize.define(NAME, {
if (minute > 420 && minute < 1140){
multiplier = 1 - Math.abs(780 - minute) / 360;
}
return multiplier;
return multiplier * house.production_multiplier;
},
unixToLocalDay: function(unix){
var house = this;
return moment.tz(unix * 1000, house.timezone).startOf('day').unix();
},
aggregatePowerToEnergyData: function(){
dayToMonthDayString: function(unix){
var house = this;
return moment.tz(unix * 1000, house.timezone).format('MM-DD');
},
aggregatePowerToEnergyData: function(){
var house = this,
base_irradiance;
return DB.EnergyDatum.destroy({where: {house_id: house.id}})
.then(()=>{
return DataHelper.baseIrradiance();
})
.then((data)=>{
base_irradiance = data;
return DB.PowerDatum.count({where: {house_id: house.id}})
})
.then((count)=>{
var limit = 0,
energy_data = new Map(),
promises = [];
console.log('House#aggregatePowerToEnergyData')
while (limit < count){
let complete = DB.PowerDatum.findAll({where: {house_id: house.id}, limit: 1000, offset: limit, order: 'id ASC'})
.then((power_data)=>{
power_data.forEach((power_datum)=>{
var day = house.unixToLocalDay(power_datum.time),
energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
energy_data.set(day, energy_datum);
});
});
.then((power_data)=>{
power_data.forEach((power_datum)=>{
var day = house.unixToLocalDay(power_datum.time),
energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
energy_data.set(day, energy_datum);
});
});
promises.push(complete);
limit += 1000;
}
return Promise.all(promises).then(()=>{
Array.from(energy_data.values()).forEach((energy_datum)=>{
let day_multiplier = 1 + Math.random() * 0.10 - 0.05,
base_day_irradiance = base_irradiance[house.dayToMonthDayString(energy_datum.day)] || 105,
irradiance = base_day_irradiance * house.production_multiplier * day_multiplier;
energy_datum.irradiance = irradiance;
})
return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true});
});
})