use c3 for stacked spline

client/config/development/style.js

@@ -1,5 +1,6 @@
 // Vendor Stylesheets
 require('bootstrap/dist/css/bootstrap.min.css');
+require('c3/c3.min.css')
 require(__dirname + '/../../d3/chart.scss');
 
 // Component Stylesheets

client/dashboard/power/graph/graph.component.js

@@ -2,7 +2,7 @@ import React from 'react';
 import Templates from 'config/templates';
 
 import House from './../../../models/house';
-import SplineStackChart from './../../../d3/line/spline_stack';
+import c3 from 'c3';
 
 class GraphComponent extends React.Component {
 
@@ -28,53 +28,38 @@ class GraphComponent extends React.Component {
 
   updateGraph(){
     var power_graph = this,
-      house = power_graph.house;
+      house = power_graph.house,
+      data = {
+          x: 'x',
+          json: house.power_data,
+          keys: {
+            x: 'time_to_date',
+            value: ['net_consumption', 'production']
+          },
+          type: 'area-spline',
+          groups: [['net_consumption', 'production']],
+          names: {
+            net_consumption: 'Net Consumption',
+            production: 'Production'
+          }
+        };
     if (power_graph.graph === undefined){
-      power_graph.graph = new SplineStackChart({
+      power_graph.chart = c3.generate({
-        container: '#power_graph',
+        bindto: '#power_graph',
-        outer_width: 800,
+        data: data,
-        outer_height: 200,
+        axis: {
-        color: '#0404B4',
+          x: {
-        time_series: true,
+            type: 'timeseries',
-        domain_attr: 'x',
+            tick: { format: d3.time.format('%d %B %y') }
-        range_attr: 'y',
+          }
-        include_dots: true,
-        titleizeDatum: (series, d)=>{
-          return series.title + '<br/>' + Math.round(d.y) + ' W<br/>' + house.formatDate(d.power_graph_datum.data.time, 'MMM D [at] HH:mm');
         }
       });
-      jQuery('#power_graph').tooltip({
+    } else {
-        selector: 'circle',
+      power_graph.chart.load({
-        container: 'body',
+        unload: true,
-        html: true,
+        data: data
-        title: function(){
-          return this.__data__.title;
-        }
       });
     }
-    var net_power_graph = {
-        title: 'Net Power Consumption',
-        values: house.power_data.map((power_graph_datum)=>{
-          return {
-            power_graph_datum: power_graph_datum,
-            x: power_graph_datum.time_to_date,
-            y: Math.max(0, power_graph_datum.data.consumption - power_graph_datum.data.production) }
-        })
-      },
-      savings = {
-        title: 'Power Production',
-        values: house.power_data.map((power_graph_datum)=>{
-          return {
-            power_graph_datum: power_graph_datum,
-            x: power_graph_datum.time_to_date,
-            y: power_graph_datum.data.production }
-        })
-      };
-    power_graph.graph.drawData({
-      title: power_graph.graph_title,
-      css_class: '',
-      series: [net_power_graph, savings]
-    });
   }
 
   render() {

client/models/power_datum.js

@@ -24,16 +24,20 @@ class PowerDatum {
     return house.toDate(power_datum.data.time);
   }
 
+  get net_consumption(){
+    return Math.max(0, Math.round(this.data.consumption - this.data.production));
+  }
+
   get time_to_s(){
     var power_datum = this,
       moment_tz = moment.tz(power_datum.data.time * 1000, power_datum.house.data.timezone);
     return moment_tz.format('YYYY-MM-DD HH:mm');
   }
-  get consumption_to_s(){
+  get consumption(){
     var power_datum = this;
     return Math.round(power_datum.data.consumption);
   }
-  get production_to_s(){
+  get production(){
     var power_datum = this;
     return Math.round(power_datum.data.production);
   }

gulpfile.babel.js

@@ -2,11 +2,12 @@ import gulp from 'gulp';
 import yargs from 'yargs';
 import webpack from 'webpack';
 import gutil from 'gulp-util';
-import gulpCopy from 'gulp-copy'
+import gulpCopy from 'gulp-copy';
 import fs from 'fs';
 
 import FsHelper from './server/lib/fs_helper';
 import ComponentMapWriter from './server/lib/tasks/component_map_writer';
+import DesignDataGenerator from './server/lib/tasks/design_data_generator';
 import BuildDashboardAssets from './server/lib/tasks/build_dashboard_assets';
 import DB from './server/config/database';
 import {PowerDataSeed, HouseSeed} from './server/lib/tasks/seed_data';
@@ -17,6 +18,19 @@ gulp.task('generate_power_csv', function(done){
   });
 });
 
+gulp.task('generate_design_data', function(){
+  var exec = require('child_process').exec,
+    house_ids = yargs.argv.house_ids.split(','),
+    start_date = parseInt(yargs.argv.start_date),
+    end_date = parseInt(yargs.argv.end_date),
+    data_generator = new DesignDataGenerator(house_ids, [start_date, end_date]);
+  return DB.sync()
+          .then(()=>{
+            return data_generator.exec();
+          });
+});
+
+
 gulp.task('save_power_csv', function(done){
   DB.sync().then(()=>{
     PowerDataSeed.saveCsv(yargs.argv, done);
@@ -53,9 +67,7 @@ gulp.task('build', function(done) {
           var path = 'client/build/design/dashboard';
           // copy all react templates and their styles sheets into build/design/dashboard.
           FsHelper.rmdirAsync(path, ()=>{
-            console.log('path removed')
             fs.mkdir(path, ()=>{
-              console.log('path recreated')
               gulp.src([
                 `client/app.scss`
               ]).pipe(gulpCopy(path, {prefix: 1}));
@@ -65,7 +77,6 @@ gulp.task('build', function(done) {
                 var files_to_copy = files.filter((file)=>{
                   return /\.(rt|scss)$/.test(file)
                 });
-                console.log(files_to_copy)
                 gulp.src(files_to_copy)
                   .pipe(gulpCopy(path, {prefix: 2}));
                 done()

package.json

@@ -34,6 +34,7 @@
     "jquery": "2.2.0",
     "bootstrap": "3.3.6",
     "d3": "3.5.12",
+    "c3": "0.4.11-rc4",
     "raw-loader": "0.5.1",
     "sass-loader": "3.1.2",
     "style-loader": "^0.12.3",
@@ -41,6 +42,7 @@
     "react-templates-loader": "0.4.0",
     "node-sass": "3.4.2",
     "moment-timezone": "0.5.0",
+    "cheerio": "0.20.0",
     "yargs": "3.32.0",
     "extend": "3.0.0",
     "through2": "2.0.1",

server/controllers/houses_controller.js

@@ -6,7 +6,7 @@ class HousesController {
 
   static index(req, res){
     var params = {};
-    if (req.query.ids) query.id = ids;
+    if (req.query.ids) params.id = ids;
     DB.House.findAll({where: params}).then((houses)=>{
       res.json({data: houses.map((house)=>{ return house.dataValues; })});
     });

server/lib/data_helper.js

@@ -0,0 +1,27 @@
+import cheerio from 'cheerio';
+import fs from 'fs';
+import moment from 'moment';
+
+class DataHelper {
+
+  static baseIrradiance(){
+    return new Promise((fnResolve, fnReject)=>{
+      fs.readFile(__dirname + '/../data/irradiance.html', (err, data)=>{
+        if (err) return fnReject(err);
+        var $ = cheerio.load(data),
+          base_irradiance = {};
+        $('tbody tr').each((i, elem)=>{
+          if (i === 0) return true;
+          var cells = $(elem).find('td'),
+            date_s = moment($(cells[1]).text(), 'YYYY-MM-DD').format('MM-DD'),
+            irradiance = parseFloat($(cells[2]).text());
+          base_irradiance[date_s] = irradiance;
+        });
+        fnResolve(base_irradiance);
+      });
+    });
+  }
+
+}
+
+export default DataHelper;

server/lib/tasks/design_data_generator.js

@@ -0,0 +1,113 @@
+import fs from 'fs';
+
+import FsHelper from './../fs_helper';
+import DB from './../../config/database';
+
+const DESIGN_DATA_PATH = __dirname + '/../../../client/build/design/data';
+
+class DesignDataGenerator {
+
+  constructor(house_ids, dates){
+    var generator = this;
+    generator.house_ids = house_ids;
+    generator.dates = dates;
+  }
+
+  exec(){
+    var generator = this;
+    console.log('Clearing design data directory...')
+    return generator.clearDirectory()
+      .then(()=>{
+        console.log('Writing house index response...')
+        return generator.writeHouseIndex();
+      })
+      .then(()=>{
+        console.log('Writing house energy and power data...');
+        return generator.writeHouseData();
+      })
+      .then(()=>{
+        console.log('Done!')
+      });
+  }
+
+  clearDirectory(){
+    console.log('DesignDataGenerator#clearDirectory')
+    return new Promise((fnResolve1, fnReject1)=>{
+      // remove directory & contents
+      FsHelper.rmdirAsync(DESIGN_DATA_PATH, ()=>{
+        // recreate it.
+        fs.mkdir(DESIGN_DATA_PATH, ()=>{
+          // create subdirectories
+          Promise.all([
+            new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/energy_data', fnResolve2); }),
+            new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/power_data', fnResolve2); })
+          ]).then(fnResolve1);
+        });
+      });
+    });
+  }
+
+  writeHouseData(){
+    var generator = this;
+    return DB.House.findAll({where: {id: generator.house_ids}})
+      .then((houses)=>{
+        var promises = [];
+        // for all houses, write energy and power responses.
+        houses.forEach((house)=>{
+          promises.push(new Promise((fnResolve, fnReject)=>{
+            DesignDataGenerator.energyIndex({house_id: house.id, dates: [generator.dates]})
+              .then((json)=>{
+                fs.writeFile(DESIGN_DATA_PATH + `/energy_data/${house.id}.json`, json, fnResolve);
+              });
+          }));
+          promises.push(new Promise((fnResolve, fnReject)=>{
+            DesignDataGenerator.powerIndex({house_id: house.id, dates: [generator.dates]})
+              .then((json)=>{
+                fs.writeFile(DESIGN_DATA_PATH + `/power_data/${house.id}.json`, json, fnResolve);
+              });
+          }));
+        });
+        return Promise.all(promises);
+      });
+  }
+
+  writeHouseIndex(){
+    var generator = this;
+    return new Promise((fnResolve, fnReject)=>{
+      DesignDataGenerator.housesIndex({id: generator.house_ids, dates: generator.dates})
+        .then((json)=>{
+          fs.writeFile(DESIGN_DATA_PATH + '/houses.json', json, fnResolve);
+        });
+    });
+  }
+
+  static housesIndex(opts){
+    return DB.House.findAll({where: {id: opts.id}})
+            .then((houses_data)=>{
+              if (opts.dates){
+                houses_data.forEach((house_datum)=>{
+                  house_datum.data_from = opts.dates[0];
+                  house_datum.data_until = opts.dates[1];
+                });
+              }
+              return JSON.stringify(houses_data, null, 2);
+            });
+  }
+
+  static powerIndex(opts){
+    return DB.PowerDatum.exposeForHouseAtDates(opts.house_id, opts.dates)
+      .then((power_data)=>{
+        return JSON.stringify({data: power_data});
+      });
+  }
+
+  static energyIndex(opts){
+    return DB.EnergyDatum.exposeForHouseAtDates(opts.house_id, opts.dates)
+      .then((energy_data)=>{
+        return JSON.stringify({data: energy_data});
+      });
+  }
+
+}
+
+export default DesignDataGenerator;

server/lib/tasks/seed_data.js

@@ -5,20 +5,22 @@ import fs from 'fs';
 import MathUtils from "./../../../shared/utils/math"
 import DB from './../../config/database';
 
-const DATA_PATH = __dirname + '/../../../shared/data/'
+const DATA_PATH = __dirname + '/../../data/'
 
 export class PowerDataSeed {
 
   static saveCsv(opts, done){
     opts = extend({
-      path: DATA_PATH + "power_data.csv"
+      path: "power_data.csv",
+      house_id: null
     }, opts || {});
+    opts.path = DATA_PATH + opts.path;
     var stream = fs.createReadStream(opts.path),
-      csvStream = csv.fromStream(stream, {headers: ['house_id', 'time', 'consumption', 'production']}),
+      csvStream = csv.fromStream(stream, {headers: ['time', 'consumption', 'production']}),
       rows = [];
 
     csvStream.on("data", function(data){
-      data.time = data.time;
+      data.house_id = opts.house_id
       rows.push(data);
       if (rows.length % 100 === 0){
         DB.PowerDatum.bulkCreate(rows, {validate: true}).catch((error)=>{
@@ -31,13 +33,8 @@ export class PowerDataSeed {
     csvStream.on("end", function(){
       console.log("all rows parsed")
       DB.PowerDatum.bulkCreate(rows, {validate: true}).then(()=>{
-        return DB.House.findAll().then((houses)=>{
+        return DB.House.findOne({where: {id: opts.house_id}}).then((house)=>{
-          var promises = [];
+          return house.aggregatePowerToEnergyData();
-          for (var house of houses){
-            var p = house.aggregatePowerToEnergyData();
-            promises.push(p);
-          }
-          return Promise.all(promises);
         });
       }).then(()=>{
         console.log("DONE!")
@@ -51,32 +48,31 @@ export class PowerDataSeed {
       end_date: moment().unix(),
       interval: 900, // every 15 minutes (in s)
       average: 1400, // Wh
-      path: DATA_PATH + "power_data.csv"
+      path: "power_data.csv"
     }, opts || {});
-console.log(opts.start_date, opts.end_date)
+    opts.path = DATA_PATH + opts.path;
+    opts.production_multiplier = parseFloat(opts.production_multiplier);
+
     var row_date = opts.start_date,
       csvStream = csv.format({headers: true}),
-      writableStream = fs.createWriteStream(opts.path),
+      writableStream = fs.createWriteStream(opts.path);
-      house_ids = opts.house_ids.split(",")
 
-    DB.House.findAll({where: {id: house_ids}}).then((houses)=>{
+    csvStream.pipe(writableStream);
+    writableStream.on("finish", ()=>{
+      console.log("DONE!")
+      done();
+    });
 
-      csvStream.pipe(writableStream);
+    DB.House.findOne({where: {id: opts.house_id}})
-      writableStream.on("finish", ()=>{
+      .then((house)=>{
-        console.log("DONE!")
+        while (row_date <= opts.end_date){
-        done();
-      });
-
-      while (row_date <= opts.end_date){
-        for (var house of houses){
           var consumption =  MathUtils.normal(opts.average),
             production = MathUtils.normal(opts.average) * house.productionMultiplier(row_date * 1000);
-          csvStream.write([house.id, row_date, consumption, production]);
+          csvStream.write([row_date, consumption, production]);
+          row_date += opts.interval;
         }
-        row_date += opts.interval;
+        csvStream.end();
-      }
+      })
-      csvStream.end();
-    });
   }
 }
 
@@ -86,7 +82,7 @@ export class HouseSeed {
       path: DATA_PATH + "houses.csv"
     }, opts || {});
     var stream = fs.createReadStream(opts.path),
-      csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'timezone']}),
+      csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'production_multiplier', 'timezone']}),
       rows = [];
 
     csvStream.on("data", function(data){

server/models/energy_datum.js

@@ -17,6 +17,7 @@ var EnergyDatum = DB.sequelize.define(NAME, {
   day: {
     type: DB.Sequelize.INTEGER,
   },
+  irradiance: DB.Sequelize.FLOAT,
   production: DB.Sequelize.FLOAT,
   consumption: DB.Sequelize.FLOAT
 }, {
@@ -34,8 +35,6 @@ var EnergyDatum = DB.sequelize.define(NAME, {
     exposeForHouseAtDates: (house_id, dates)=>{
       var params = {house_id: house_id};
       extend(params, ApiHelper.datesParamToSequelize(dates, 'day'));
-      console.log('EnergyDatum#exposeForHouseAtDates')
-      console.log(params, dates)
       return EnergyDatum.findAll({
         where: params,
         attributes: ['id', 'production', 'consumption', 'day']

server/models/house.js

@@ -1,5 +1,6 @@
 import moment from 'moment-timezone';
 import DB from "./../config/database";
+import DataHelper from './../lib/data_helper';
 
 const NAME = 'House';
 
@@ -15,6 +16,7 @@ var House = DB.sequelize.define(NAME, {
   },
   timezone: DB.Sequelize.STRING,
   name: DB.Sequelize.STRING,
+  production_multiplier: DB.Sequelize.FLOAT,
   data_until: {
     type: DB.Sequelize.INTEGER,
   },
@@ -33,37 +35,53 @@ var House = DB.sequelize.define(NAME, {
       if (minute > 420 && minute < 1140){
         multiplier = 1 - Math.abs(780 - minute) / 360;
       }
-      return multiplier;
+      return multiplier * house.production_multiplier;
     },
     unixToLocalDay: function(unix){
       var house = this;
       return moment.tz(unix * 1000, house.timezone).startOf('day').unix();
     },
-    aggregatePowerToEnergyData: function(){
+    dayToMonthDayString: function(unix){
       var house = this;
+      return moment.tz(unix * 1000, house.timezone).format('MM-DD');
+    },
+    aggregatePowerToEnergyData: function(){
+      var house = this,
+        base_irradiance;
       return DB.EnergyDatum.destroy({where: {house_id: house.id}})
         .then(()=>{
+          return DataHelper.baseIrradiance();
+        })
+        .then((data)=>{
+          base_irradiance = data;
           return DB.PowerDatum.count({where: {house_id: house.id}})
         })
         .then((count)=>{
           var limit = 0,
             energy_data = new Map(),
             promises = [];
+          console.log('House#aggregatePowerToEnergyData')
           while (limit < count){
             let complete = DB.PowerDatum.findAll({where: {house_id: house.id}, limit: 1000, offset: limit, order: 'id ASC'})
-                    .then((power_data)=>{
+              .then((power_data)=>{
-                      power_data.forEach((power_datum)=>{
+                power_data.forEach((power_datum)=>{
-                        var day = house.unixToLocalDay(power_datum.time),
+                  var day = house.unixToLocalDay(power_datum.time),
-                          energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
+                    energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
-                        energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
+                  energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
-                        energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
+                  energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
-                        energy_data.set(day, energy_datum);
+                  energy_data.set(day, energy_datum);
-                      });
+                });
-                    });
+              });
             promises.push(complete);
             limit += 1000;
           }
           return Promise.all(promises).then(()=>{
+            Array.from(energy_data.values()).forEach((energy_datum)=>{
+              let day_multiplier = 1 + Math.random() * 0.10 - 0.05,
+                base_day_irradiance = base_irradiance[house.dayToMonthDayString(energy_datum.day)] || 105,
+                irradiance = base_day_irradiance * house.production_multiplier * day_multiplier;
+              energy_datum.irradiance = irradiance;
+            })
             return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true});
           });
         })