fix problems with rendering data

client/api/energy_data.js

@@ -5,7 +5,7 @@ const ENDPOINT = '/data/v1/energy';
 // send all date parameters as unix timestamps;
 class EnergyDataApi {
 
-  static index(params){
+  static index(house, params){
     params = extend({
     }, params);
     if (params.dates){

client/api/houses.js

@@ -1,6 +1,7 @@
-const ENDPOINT = '/data/v1/houses';
 import extend from 'extend';
 
+const ENDPOINT = '/data/v1/houses';
+
 class HousesApi {
 
   static index(params){

client/api/power_data.js

@@ -5,15 +5,6 @@ import extend from 'extend';
 class PowerDataApi {
 
   static index(params){
-    params = extend({
-    }, params);
-    if (params.dates){
-      params.dates = params.dates.map((date_range)=>{
-        if (date_range[0]) date_range[0] = date_range[0].unix();
-        if (date_range[1]) date_range[1] = date_range[1].unix();
-        return [date_range[0], date_range[1]];
-      })
-    }
     return jQuery.ajax({
       url: ENDPOINT  + '?' + jQuery.param(params),
       type: 'GET',

client/d3/line/spline_stack.js

@@ -30,10 +30,8 @@ class SplineStackChart extends LineChart {
     var spline_stack = this,
       serialized_data = {
         series: [] };
-
     data.series.forEach(function(series, i){
-      series.css_class = series.css_class || spline_stack.toClass ? spline_stack.toClass(series) : "";
+      series.css_class = series.css_class || spline_stack.toClass ? spline_stack.toClass(series) : "series-" + i;
-      series.title = series.title || spline_stack.toClass ? spline_stack.titleize(series) : "";
       if (spline_stack.domain_attr !== 'x' && spline_stack.range_attr !== 'y'){
         series.values = series.values.map((value)=>{
           return {x: value[spline_stack.domain_attr], y: value[spline_stack.range_attr], series: series};
@@ -41,6 +39,7 @@ class SplineStackChart extends LineChart {
       }
       serialized_data.series.push(series);
     });
+
     serialized_data.series = spline_stack.fnStack(serialized_data.series);
     // assume all series have same domain, use first series to establish extent.
     serialized_data.domain_extent = d3.extent(serialized_data.series[0].values.map((value)=>{ return value.x; }));
@@ -68,6 +67,17 @@ class SplineStackChart extends LineChart {
       spline_stack.applyData(paths);
     });
     stack.exit().remove();
+
+    if (spline_stack.include_dots){
+      data.series.forEach((series)=>{
+        var dots = spline_stack.svg.selectAll(".d3-chart-spline-dot." + series.css_class)
+            .data(series.values);
+        [dots.enter().append("circle"), dots.transition()].forEach((circles)=>{
+          spline_stack.applyDots(series, circles);
+        });
+        dots.exit().remove();
+      });
+    }
   }
 
   applyData(paths){
@@ -75,8 +85,24 @@ class SplineStackChart extends LineChart {
     paths
       .attr("class", function(series){"d3-chart-spline-stack " + series.css_class;})
       .attr("d", function(series){ return spline_stack.fnArea(series.values); })
-      .style("fill", function(series){ return spline_stack.fnColor(series.title); });
+      .style("fill", function(series){ return spline_stack.fnColor(series.title); })
+      .attr('opacity', 1);
   }
+
+  applyDots(series, circles){
+    var spline_stack = this;
+    circles
+      .attr('class', 'd3-chart-spline-dot' + series.css_class)
+      .attr("r", 2)
+      .attr("cx", function(d, i){ return spline_stack.x_scale(d.x); })
+      .attr("cy", function(d, i){ return spline_stack.y_scale(d.y + d.y0); })
+      .attr("title", function(d, i){ return spline_stack.titleizeDatum(series, d); })
+      .style("fill", spline_stack.fnColor(series.title))
+      .attr('opacity', 1)
+      .attr('stroke-width', 0)
+      .attr('stroke', '#fff');
+  }
+
 }
 
 export default SplineStackChart;

client/d3/sliders/date_range.js

@@ -0,0 +1,133 @@
+import Chart from './../base';
+
+class DateRange extends Chart {
+
+
+  get chart_options(){
+    return Object.assign(Object.assign({}, Chart.DEFAULTS), {
+      outer_width: 600,
+      outer_height: 250,
+      margin: {top: 20, left: 10, bottom: 20, right: 10},
+    });
+  }
+
+  defineAxes(){
+    var date_range = this;
+
+    date_range.x_scale = d3.time.scale()
+      .range([0, date_range.width])
+      .clamp(true);
+
+    date_range.x_axis = d3.svg.axis()
+      .scale(date_range.x_scale)
+      .orient("bottom")
+      .ticks(d3.time.weeks, 1)
+      //.tickFormat(function(d) { return d + "°"; })
+      .tickSize(1)
+      .outerTickSize(1)
+      .tickPadding(12)
+
+    date_range.svg.append("g")
+        .attr("class", "d3-chart-domain")
+        .attr("transform", "translate(0," + date_range.height / 2 + ")");
+  }
+
+  afterAxes(){
+    var date_range = this;
+
+    date_range.slider = date_range.svg.append("g")
+        .attr("class", "d3-chart-slider");
+
+    date_range.min_handle = date_range.slider.append("circle")
+      .attr("class", "d3-chart-min-handle")
+      .attr("transform", "translate(0," + date_range.height / 2 + ")")
+      .attr("r", 9);
+
+    date_range.max_handle = date_range.slider.append("circle")
+      .attr("class", "d3-chart-max-handle")
+      .attr("transform", "translate(0," + date_range.height / 2 + ")")
+      .attr("r", 9);
+
+    date_range.brush = d3.svg.brush()
+      .x(date_range.x_scale);
+
+    date_range.slider
+      .call(date_range.brush)
+      //.select(".background")
+      //  .attr("height", date_range.height);
+
+    date_range.slider.call(date_range.brush)
+      .selectAll(".extent,.resize")
+      .remove();
+  }
+
+  drawData(data){
+    var date_range = this;
+    date_range.x_scale.domain([data.abs_min, data.abs_max]);
+
+    date_range.svg.select(".d3-chart-domain")
+      .call(date_range.x_axis);
+
+    date_range.min_handle.attr('cx', date_range.x_scale(data.current_min));
+    date_range.max_handle.attr('cx', date_range.x_scale(data.current_max));
+
+    date_range.brush.extent([data.current_min, data.current_min])
+      .on("brush", ()=>{
+        DateRange.handleBrush(date_range, eval('this'));
+      });
+
+    date_range.slider
+        .call(date_range.brush.event)
+      .transition() // gratuitous intro!
+        .duration(750)
+        .call(date_range.brush.extent([data.current_min, data.current_min]))
+        .call(date_range.brush.event);
+
+  }
+
+  static handleBrush(date_range, elem){
+    var date = date_range.brush.extent()[0],
+      current_min = parseInt(date_range.min_handle.attr('cx')),
+      current_max = parseInt(date_range.max_handle.attr('cx'));
+
+    if (!current_min && !current_max) return false
+    if (d3.event.sourceEvent) { // not a programmatic event
+      date = date_range.x_scale.invert(d3.mouse(elem)[0]);
+      date_range.brush.extent([date, date]);
+    }
+
+    var value = date_range.x_scale(date);
+
+    if (value < current_max && value > current_min){
+      if (Math.abs(value - current_min) < Math.abs(value - current_max)){
+        date_range.min_handle.attr('cx', value);
+        current_min = value;
+      } else {
+        date_range.max_handle.attr('cx', value);
+        current_max = value;
+      }
+    } else if (value >= current_max){
+      date_range.max_handle.attr('cx', value);
+      current_max = value;
+      if (d3.event.sourceEvent && date_range.maxDelta){
+        var new_current_min = date_range.maxDelta(date, date_range.x_scale.invert(current_min));
+        if (new_current_min) date_range.min_handle.attr('cx', date_range.x_scale(new_current_min));
+      }
+    } else {
+      date_range.min_handle.attr('cx', value);
+      current_min = value;
+      if (d3.event.sourceEvent && date_range.maxDelta){
+        var new_current_max = date_range.maxDelta(date, date_range.x_scale.invert(current_max));
+        if (new_current_max) date_range.max_handle.attr('cx', date_range.x_scale(new_current_max));
+      }
+    }
+
+
+    if (d3.event.sourceEvent && date_range.onRangeUpdated) {
+      date_range.onRangeUpdated(date_range.x_scale.invert(current_min), date_range.x_scale.invert(current_max));
+    }
+  }
+
+}
+
+export default DateRange;

client/dashboard/energy/energy.js

@@ -22,7 +22,7 @@ var Energy = React.createClass({
     var energy = this,
       house = energy.props.house;
     energy.graph_title = 'Daily Consumption';
-    house.ensureEnergyData().then(()=>{
+    house.setEnergyData().then(()=>{
       energy.setState({loading_data: false});
       if (energy.props.view === 'graph') energy.initGraph();
     });
@@ -32,7 +32,7 @@ var Energy = React.createClass({
     var energy = this;
     if (new_props.house !== energy.state.house){
       energy.setState({loading_data: true});
-      new_props.house.ensureEnergyData().then(()=>{
+      new_props.house.setEnergyData().then(()=>{
         energy.setState({loading_data: false});
         if (energy.props.view === 'graph') energy.initGraph();
       });

client/dashboard/energy/energy.rt

@@ -27,7 +27,7 @@
       </tr>
     </thead>
     <tbody>
-      <tr rt-repeat="energy_datum in this.props.house.energy_data" key="{energy_datum.react_key}">
+      <tr rt-repeat="energy_datum in this.props.house.energy_data" key="{energy_datum.scoped_id}">
         <td></td>
         <td>{energy_datum.day_to_s}</td>
         <td>{energy_datum.consumption_to_s}</td>

client/dashboard/energy/energy.rt.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import _ from 'lodash';
 function repeatEnergy_datum1(energy_datum, energy_datumIndex) {
-    return React.createElement('tr', { 'key': energy_datum.react_key }, React.createElement('td', {}), React.createElement('td', {}, energy_datum.day_to_s), React.createElement('td', {}, energy_datum.consumption_to_s), React.createElement('td', {}, energy_datum.production_to_s));
+    return React.createElement('tr', { 'key': energy_datum.scoped_id }, React.createElement('td', {}), React.createElement('td', {}, energy_datum.day_to_s), React.createElement('td', {}, energy_datum.consumption_to_s), React.createElement('td', {}, energy_datum.production_to_s));
 }
 export default function () {
     return React.createElement('div', { 'id': 'energy_view' }, this.state.loading_data ? React.createElement('div', { 'className': 'alert alert-warning' }, '\n    Retrieving energy data for the ', this.props.house.name, ' household...\n  ') : null, this.props.view === 'graph' ? React.createElement('div', {}, React.createElement('h4', {}, 'Select Data'), React.createElement('div', {

client/dashboard/layout/layout.js

@@ -1,6 +1,7 @@
 import React from 'react';
 import layoutRt from './layout.rt.js';
 import House from './../../models/house';
+import PowerDatum from './../../models/power_datum';
 
 var Layout = React.createClass({
 
@@ -31,7 +32,9 @@ var Layout = React.createClass({
     var layout = this,
       house_id = event.target.value,
       house = layout.state.houses.find((house)=>{ return house.data.id == house_id });
-    layout.setState({house: house});
+    layout.setState({house: house}, ()=>{
+      house.closeDb();
+    });
   },
 
   setView: function(event){
@@ -47,6 +50,19 @@ var Layout = React.createClass({
     layout.setState({dataset: dataset});
   },
 
+  refreshData: function(){
+    var layout = this,
+      houses = layout.state.houses,
+      all = [];
+    houses.forEach((house)=>{
+      all.push(house.clearData());
+    });
+    Promise.all(all)
+      .then(()=>{
+        window.location.reload();
+      });
+  },
+
   render: function() {
     return layoutRt.call(this);
   }

client/dashboard/layout/layout.rt

@@ -2,12 +2,12 @@
 <rt-require dependency="./../power/power" as="Power"/>
 <div id="layout">
   <div class="alert alert-warning" rt-if="this.state.requesting_data">Retrieving houses...</div>
-  <h1 rt-if="this.state.house">{this.state.house.name}</h1>
 
   <h4>Select household:</h4>
   <select rt-if="this.state.houses" class="form-control" onChange="{this.setHouse}">
-    <option rt-repeat="house in this.state.houses" value="{house.data.id}" key="{house.react_key}">{house.data.name}</option>
+    <option rt-repeat="house in this.state.houses" value="{house.data.id}" key="{house.scoped_id}">{house.data.name}</option>
   </select>
+  <button rt-if="this.state.house" onClick="{this.refreshData}" class="btn btn-xs btn-default">Refresh House Data</button>
 
   <h4>Select dataset:</h4>
   <div class="btn-group" role="group">

client/dashboard/layout/layout.rt.js

@@ -5,18 +5,21 @@ import Power from './../power/power';
 function repeatHouse1(house, houseIndex) {
     return React.createElement('option', {
         'value': house.data.id,
-        'key': house.react_key
+        'key': house.scoped_id
     }, house.data.name);
 }
 export default function () {
-    return React.createElement('div', { 'id': 'layout' }, this.state.requesting_data ? React.createElement('div', { 'className': 'alert alert-warning' }, 'Retrieving houses...') : null, this.state.house ? React.createElement('h1', {}, this.state.house.name) : null, React.createElement('h4', {}, 'Select household:'), this.state.houses ? React.createElement.apply(this, [
+    return React.createElement('div', { 'id': 'layout' }, this.state.requesting_data ? React.createElement('div', { 'className': 'alert alert-warning' }, 'Retrieving houses...') : null, React.createElement('h4', {}, 'Select household:'), this.state.houses ? React.createElement.apply(this, [
         'select',
         {
             'className': 'form-control',
             'onChange': this.setHouse
         },
         _.map(this.state.houses, repeatHouse1.bind(this))
-    ]) : null, React.createElement('h4', {}, 'Select dataset:'), React.createElement('div', {
+    ]) : null, this.state.house ? React.createElement('button', {
+        'onClick': this.refreshData,
+        'className': 'btn btn-xs btn-default'
+    }, 'Refresh House Data') : null, React.createElement('h4', {}, 'Select dataset:'), React.createElement('div', {
         'className': 'btn-group',
         'role': 'group'
     }, React.createElement('button', {

client/dashboard/power/power.js

@@ -1,7 +1,11 @@
 import React from 'react';
+import moment from 'moment-timezone';
+import _ from 'lodash';
+
 import powerRt from './power.rt.js';
 import House from './../../models/house';
 import SplineStackChart from './../../d3/line/spline_stack';
+import DateRangeSlider from './../../d3/sliders/date_range';
 
 var Power = React.createClass({
 
@@ -21,45 +25,63 @@ var Power = React.createClass({
     var power = this,
       house = power.props.house;
     power.graph_title = '';
-    house.ensurePowerData().then(()=>{
+    power.initDateRange();
+    house.setPowerData().then(()=>{
       power.setState({loading_data: false});
-      if (power.props.view === 'graph') power.initGraph();
+      if (power.props.view === 'graph'){
+        power.initGraph();
+      }
     });
   },
 
   componentWillReceiveProps: function(new_props){
     var power = this;
     if (new_props.house !== power.state.house){
+      // house will change.
       power.setState({loading_data: true});
-      new_props.house.ensurePowerData().then(()=>{
+      new_props.house.setPowerData().then(()=>{
         power.setState({loading_data: false});
-        if (power.props.view === 'graph') power.initGraph();
+        if (power.props.view === 'graph'){
+          power.initDateRange();
+          power.initGraph();
+        }
       });
     }
+    // view will change from graph to table.
     if (new_props.view !== 'graph' && power.props.view === 'graph') power.destroyGraph();
   },
 
   componentDidUpdate: function(prev_props, _prev_state){
     var power = this,
       house = power.props.house;
-    if (prev_props.view !== 'graph' && power.props.view === 'graph') power.initGraph();
+    // view has changed from graph to table.
+    if (prev_props.view !== 'graph' && power.props.view === 'graph'){
+      power.initDateRange();
+      power.initGraph();
+    }
   },
 
   initGraph: function(){
-    var power = this;
+    var power = this,
+      house = power.props.house;
     if (power.graph === undefined){
       power.graph = new SplineStackChart({
         container: '#power_graph',
         outer_width: 800,
         outer_height: 200,
         color: '#0404B4',
-        range_attr: 'y',
+        time_series: true,
         domain_attr: 'x',
-        time_series: true
+        range_attr: 'y',
+        include_dots: true,
+        titleizeDatum: (series, d)=>{
+          return series.title + '<br/>' + Math.round(d.y) + ' W<br/>' + moment.tz(d.x.getTime(), house.timezone).format('MMM D [at] HH:mm')
+        }
       });
       jQuery('#power_graph').tooltip({
-        selector: 'path',
+        selector: 'circle',
         container: 'body',
+        html: true,
         title: function(){
           return this.__data__.title;
         }
@@ -73,11 +95,19 @@ var Power = React.createClass({
       house = power.props.house,
       net_power = {
         title: 'Net Power Consumption',
-        values: house.power_data.slice(0, 200).map((power_datum)=>{ return {y: Math.max(0, power_datum.data.consumption - power_datum.data.production), x: power_datum.data.time.toDate() } })
+        values: house.power_data.map((power_datum)=>{
+          return {
+            x: house.toDate(power_datum.data.time),
+            y: Math.max(0, power_datum.data.consumption - power_datum.data.production) }
+        })
       },
       savings = {
         title: 'Power Production',
-        values: house.power_data.slice(0, 200).map((power_datum)=>{ return {y: power_datum.data.production, x: power_datum.data.time.toDate() } })
+        values: house.power_data.map((power_datum)=>{
+          return {
+            x: house.toDate(power_datum.data.time),
+            y: power_datum.data.production }
+        })
       };
     power.graph.drawData({
       title: power.graph_title,
@@ -86,9 +116,50 @@ var Power = React.createClass({
     });
   },
 
+  initDateRange: function(){
+    var power = this,
+      house = power.props.house;
+    if (power.date_range_slider === undefined){
+      power.date_range_slider = new DateRangeSlider({
+        container: '#power_date_setter',
+        outer_height: 100,
+        maxDelta: function(changed_date, other_date){
+          if (Math.abs(changed_date.getTime() - other_date.getTime()) > 3600 * 24 * 7 * 1000){
+            if (changed_date > other_date){
+              return new Date(changed_date.getTime() - 3600 * 24 * 4 * 1000);
+            } else {
+              return new Date(changed_date.getTime() + 3600 * 24 * 4 * 1000);
+            }
+          }
+          return false;
+        }
+      });
+    }
+    power.date_range_slider.onRangeUpdated = (min, max)=>{
+      if (power.date_range_update) clearTimeout(power.date_range_update)
+      power.date_range_update = setTimeout(()=>{
+        house.power_date_range = [Math.round(min.getTime() / 1000), Math.round(max.getTime() / 1000)]
+        house.setPowerData()
+          .then(()=>{
+            power.updateGraph();
+          });
+      }, 500);
+    };
+    var four_week_start = house.data.data_until - 3600 * 24 * 28,
+      data_from = Math.max(four_week_start, house.data.data_from);
+    power.date_range_slider.drawData({
+      abs_min: house.toDate(four_week_start),
+      abs_max: house.toDate(house.data.data_until),
+      current_min: house.toDate(house.default_power_start),
+      current_max: house.toDate(house.default_power_end)
+    });
+  },
+
   destroyGraph: function(){
     var power = this;
+    document.getElementById('power_date_setter').innerHTML = '';
     document.getElementById('power_graph').innerHTML = '';
+    power.date_range_slider = undefined;
     power.graph = undefined;
   },
 

client/dashboard/power/power.rt

@@ -1,8 +1,9 @@
 <div id="power_view">
-  <h2>Household 15-minute Power Statistics</h2>
   <div class="alert alert-warning" rt-if="this.state.loading_data">
     Retrieving power data for the {this.props.house.name} household...
   </div>
+  <h4>Select dates:</h4>
+  <div id="power_date_setter"></div>
   <table rt-if="this.props.view === 'table'" class="table">
     <thead>
       <tr>
@@ -13,7 +14,7 @@
       </tr>
     </thead>
     <tbody>
-      <tr rt-repeat="power_datum in this.props.house.power_data" key="{power_datum.react_key}">
+      <tr rt-repeat="power_datum in this.props.house.power_data" key="{power_datum.scoped_id}">
         <td></td>
         <td>{power_datum.time_to_s}</td>
         <td>{power_datum.consumption_to_s}</td>

client/dashboard/power/power.rt.js

@@ -1,10 +1,10 @@
 import React from 'react';
 import _ from 'lodash';
 function repeatPower_datum1(power_datum, power_datumIndex) {
-    return React.createElement('tr', { 'key': power_datum.react_key }, React.createElement('td', {}), React.createElement('td', {}, power_datum.time_to_s), React.createElement('td', {}, power_datum.consumption_to_s), React.createElement('td', {}, power_datum.production_to_s));
+    return React.createElement('tr', { 'key': power_datum.scoped_id }, React.createElement('td', {}), React.createElement('td', {}, power_datum.time_to_s), React.createElement('td', {}, power_datum.consumption_to_s), React.createElement('td', {}, power_datum.production_to_s));
 }
 export default function () {
-    return React.createElement('div', { 'id': 'power_view' }, React.createElement('h2', {}, 'Household 15-minute Power Statistics'), this.state.loading_data ? React.createElement('div', { 'className': 'alert alert-warning' }, '\n    Retrieving power data for the ', this.props.house.name, ' household...\n  ') : null, this.props.view === 'table' ? React.createElement('table', { 'className': 'table' }, React.createElement('thead', {}, React.createElement('tr', {}, React.createElement('th', {}), React.createElement('th', {}, 'Time'), React.createElement('th', {}, 'Consumption (W)'), React.createElement('th', {}, 'Production (W)'))), React.createElement.apply(this, [
+    return React.createElement('div', { 'id': 'power_view' }, this.state.loading_data ? React.createElement('div', { 'className': 'alert alert-warning' }, '\n    Retrieving power data for the ', this.props.house.name, ' household...\n  ') : null, React.createElement('h4', {}, 'Select dates:'), React.createElement('div', { 'id': 'power_date_setter' }), this.props.view === 'table' ? React.createElement('table', { 'className': 'table' }, React.createElement('thead', {}, React.createElement('tr', {}, React.createElement('th', {}), React.createElement('th', {}, 'Time'), React.createElement('th', {}, 'Consumption (W)'), React.createElement('th', {}, 'Production (W)'))), React.createElement.apply(this, [
         'tbody',
         {},
         _.map(this.props.house.power_data, repeatPower_datum1.bind(this))

client/lib/databasable.js

@@ -1,5 +1,5 @@
-import Loki from 'lokijs';
+import Loki from 'lokijs/src/lokijs';
-import LokiIndexedAdapter from 'loki/loki-indexed-adapter';
+import LokiIndexedAdapter from 'lokijs/src/loki-indexed-adapter';
 
 const DEFAULTS = {
   autosave: false
@@ -10,16 +10,16 @@ var databasable = {
   accessDb: function(db_name, opts){
     var databasable = this;
     opts = Object.assign(Object.assign({
-      persistenceMethod: 'adapter',
+      adapter: new LokiIndexedAdapter(db_name)
-      persistenceAdapter: new LokiIndexedAdapter(db_name)
     }, DEFAULTS), opts || {});
-    return new Promise((fnResolve, fnReject){
+    return new Promise((fnResolve, fnReject)=>{
       if (!databasable.db) {
         databasable.db = new Loki(db_name, opts);
         databasable.db.loadDatabase({}, ()=>{
           fnResolve(databasable.db);
         });
       } else { fnResolve(databasable.db); }
+
     });
   },
 
@@ -35,14 +35,19 @@ var databasable = {
   collection: function(collection_name, options){
     var databasable = this;
     return databasable.accessDb()
-      .then((db)=>{
+        .then((db)=>{
-        var collection = db.getCollection(collection_name)
+          var collection = db.getCollection(collection_name)
-        if (!collection){
+          if (!collection){
-          collection = db.addCollection(collection_name, options);
+            collection = db.addCollection(collection_name, options);
-        }
+          }
-        return collection;
+          if (options && options.unique_indices){
-      });
+            options.unique_indices.forEach((field)=>{
-  }
+              collection.ensureUniqueIndex(field);
+            });
+          }
+          return collection;
+        });
+  },
 
   rangeToLokiParams: function(attr, range){
     var date_params = {};
@@ -50,7 +55,7 @@ var databasable = {
       var start_condition = {},
         end_condition = {};
       date_params['$and'] = [start_condition, end_condition];
-      start_condition[attr] = {'$lt': range[1]};
+      start_condition[attr] = {'$gt': range[0]};
       end_condition[attr] = {'$lt': range[1]};
     } else if (range[0] !== undefined) {
       date_params[attr] = {'$gt': range[0]}
@@ -62,4 +67,4 @@ var databasable = {
 
 };
 
-export default databaseable;
+export default databasable;

client/models/energy_datum.js

@@ -12,7 +12,6 @@ class EnergyDatum {
     energy_datum.house = house;
     data.day = moment.tz(data.day, house.data.timezone);
     energy_datum.data = data;
-    EnergyDatum.store.set(data.id, energy_datum);
   }
 
   get react_key(){

client/models/house.js

@@ -1,5 +1,6 @@
 import extend from 'extend';
-import Loki from 'loki';
+import Loki from 'lokijs/src/lokijs';
+import moment from 'moment-timezone';
 
 import PowerDatum from './power_datum';
 import EnergyDatum from './energy_datum';
@@ -18,30 +19,57 @@ class House {
     var house = this;
     house.data = data;
     Object.assign(house, Databasable);
+    house.power_date_range = [house.default_power_start, house.default_power_end];
   }
 
   get scoped_id(){
     return `house-${this.data.id}`;
   }
 
+  get default_power_start(){
+    var house = this;
+    // 3600 * 24 seconds * 4 = 4 days.
+    return house.data.data_until - 3600 * 24 * 4;
+  }
+
+  get default_power_end(){
+    var house = this;
+    return house.data.data_until;
+  }
+
+  toDate(unix){
+    var house = this;
+    return moment.tz(unix * 1000, house.data.timezone).toDate();
+  }
+
   save(){
     var house = this;
-    return House.collection()
+    return House.collection(House.NAME)
       .then((house_collection)=>{
-        return house_collection.update(house.data);
+        house_collection.update(house.data);
+        return House.db.save();
       });
   }
 
   setPowerData(opts){
     var house = this;
-    return house.ensurePowerData(opts)
+    opts = Object.assign({
-      .then(()=>{
+      dates: house.power_date_range
-        return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS)
+    }, opts || {});
-          .then((power_collection)=>{
+    return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS)
-            var params = house.rangeToLokiParams('time', [opts.start_date, opts.end_date]);
+      .then((power_collection)=>{
-            house.power_data = power_collection.find(params).map((data)=>{ return new PowerDatum(data); })
+        return house.ensurePowerData(opts)
+          .then(()=>{
+            var params = house.rangeToLokiParams('time', opts.dates);
+            house.power_data = power_collection.find(params)
+                  .sort((pd1, pd2)=>{
+                    if (pd1.time === pd2.time) return 0;
+                    if (pd1.time > pd2.time) return 1;
+                    if (pd1.time < pd2.time) return -1;
+                  })
+                  .map((data)=>{ return new PowerDatum(data, house); })
           });
-      })
+      });
   }
 
   ensurePowerData(opts){
@@ -50,13 +78,16 @@ class House {
       end_date: undefined
     }, opts || {});
     var house = this,
-      query_ranges = DateRange.addRange([opts.start_date, opts.end_date], house.data.power_datum_ranges);
+      existing_ranges = house.data.power_datum_ranges || [],
+      query_ranges;
 
+    query_ranges = DateRange.addRange(opts.dates, existing_ranges);
     if (query_ranges.gaps_filled.length > 0){
-      house.getPowerData({dates: query_ranges.gaps_filled})
+      var params = {dates: query_ranges.gaps_filled};
+      return house.getPowerData(params)
         .then(()=>{
           house.data.power_datum_ranges = query_ranges.new_ranges;
-          return house.save();
+          house.save();
         });
     } else { return Promise.resolve(); }
   }
@@ -66,14 +97,33 @@ class House {
     params.house_id = house.data.id;
     return PowerDataApi.index(params)
       .then((power_data)=>{
-        return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS);
+        return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS)
                 .then((power_collection)=>{
                   power_collection.insert(power_data);
-                  return house.db.save();
+                  house.db.save();
                 });
       })
   }
 
+  clearData(){
+    var house = this;
+    return new Promise((fnResolve, fnReject)=>{
+      house.collection(PowerDatum.NAME)
+        .then((power_collection)=>{
+          power_collection.removeWhere({});
+          house.db.save(()=>{
+            House.collection(House.NAME)
+              .then((house_collection)=>{
+                house_collection.remove(house.data);
+                House.db.save(()=>{
+                  fnResolve();
+                })
+              });
+          });
+        });
+    });
+  }
+
   setEnergyData(opts){
     var house = this;
     return house.ensureEnergyData(opts)
@@ -81,7 +131,7 @@ class House {
         return house.collection(EnergyDatum.NAME, EnergyDatum.COLLECTION_OPTIONS)
           .then((energy_collection)=>{
             var params = house.rangeToLokiParams('day', [opts.start_date, opts.end_date]);
-            house.energy_data = energy_collection.find(params).map((data)=>{ return new EnergyDatum(data); })
+            house.energy_data = energy_collection.find(params).map((data)=>{ return new EnergyDatum(data, house); })
           });
       })
   }
@@ -108,7 +158,7 @@ class House {
     params.house_id = house.data.id;
     return EnergyDataApi.index(params)
       .then((energy_data)=>{
-        return house.collection(EnergyDatum.NAME, EnergyDatum.COLLECTION_OPTIONS);
+        return house.collection(EnergyDatum.NAME, EnergyDatum.COLLECTION_OPTIONS)
                 .then((energy_collection)=>{
                   energy_collection.insert(energy_data);
                   return house.db.save();
@@ -117,34 +167,25 @@ class House {
   }
 
   static ensureHouses(ids){
-    House.collection()
+    return House.collection(House.NAME)
       .then((house_collection)=>{
-        houses = house_collection.find({id: {$in: ids}});
+        var houses_data = ids ? house_collection.find({id: {$in: ids}}) : house_collection.find();
-        if (houses.length !== ids.length){
+        if (!ids && houses_data.length === 0 || ids && houses_data.length !== ids.length){
-          required_ids = ArrayUtil.diff(ids, houses.map((house)=>{ return house.id; }));
+          var required_ids = ids ? ArrayUtil.diff(ids, houses_data.map((data)=>{ return data.id; })) : undefined;
-          return House.getHouses(required_ids)
+          return HousesApi.index({id: ids})
-            .then((required_houses){
+            .then((required_houses)=>{
-              return houses.concat(required_houses);
+              required_houses.forEach((house_data)=>{
+                house_collection.insert(house_data);
+              });
+              House.db.save();
+              return houses_data.concat(required_houses);
             });
-        } else { return houses; }
+        } else { return Promise.resolve(houses_data); }
-      }).then((house_data)=>{
+      }).then((houses_data)=>{
         return houses_data.map((house_data)=>{ return new House(house_data); })
       });
   }
 
-  static getHouses(ids){
-    return HousesApi.index({id: ids})
-      .then((houses_data)=>{
-        return House.collection()
-          .then((house_collection)=>{
-            houses_data.forEach((house_data)=>{
-              house_collection.insert(house_data);
-            });
-            return houses_data;
-          });
-      });
-  }
-
 }
 
 Object.assign(House, Databasable);

client/models/power_datum.js

@@ -3,16 +3,15 @@ import moment from 'moment-timezone';
 
 const NAME = 'PowerDatum';
 const COLLECTION_DEFAULTS = {
-  indices: ['time']
+  indices: ['time'],
+  unique_indices: ['time']
 };
 
 class PowerDatum {
   constructor(data, house){
     var power_datum = this;
     power_datum.house = house;
-    data.time = moment.tz(data.time, house.data.timezone);
     power_datum.data = data;
-    PowerDatum.store.set(data.id, power_datum);
   }
 
   get react_key(){

server/config/webpack/development.js

@@ -41,5 +41,8 @@ module.exports = {
           d3: "d3",
           "window.d3": "d3"
       })
-  ]
+  ],
+  node: {
+    fs: "empty"
+  }
 }

server/config/webpack/production.js

@@ -51,5 +51,8 @@ module.exports = {
           jQuery: "jquery",
           "window.jQuery": "jquery"
       })
-  ]
+  ],
+  node: {
+    fs: "empty"
+  }
 };

server/controllers/houses_controller.js

@@ -7,7 +7,7 @@ class HousesController {
   static index(req, res){
     var params = {};
     if (req.query.ids) query.id = ids;
-    DB.House.findAll({where: params, attributes: ['id', 'name', 'timezone']}).then((houses)=>{
+    DB.House.findAll({where: params}).then((houses)=>{
       res.json({data: houses.map((house)=>{ return house.dataValues; })});
     });
   }

server/controllers/power_controller.js

@@ -5,7 +5,6 @@ const NAME = 'PowerController';
 class PowerController{
 
   static index(req, res){
-    console.log(req.query);
     DB.PowerDatum.exposeForHouseAtDates(req.query.house_id, req.query.dates).then((power_data)=>{
       res.json({data: power_data});
     });

server/helpers/api_helper.js

@@ -1,5 +1,3 @@
-import moment from 'moment';
-
 class ApiHelper {
 
   // assume all dates from api coming as UNIX timestamps.
@@ -12,15 +10,15 @@ class ApiHelper {
       dates.forEach((min_max)=>{
         var condition_n = {};
         condition_n[field_name] = {};
-        if (min_max[0]) condition_n[field_name]['$gt'] = moment.unix(min_max[0]).toDate();
+        if (min_max[0]) condition_n[field_name]['$gt'] = min_max[0];
-        if (min_max[1]) condition_n[field_name]['$lt'] = moment.unix(min_max[1]).toDate();
+        if (min_max[1]) condition_n[field_name]['$lt'] = min_max[1];
         if (Object.keys(condition_n).length) params['$or'].push(condition_n);
       });
     } else {
       var min_max = dates[0],
         condition = {}
-      if (min_max[0]) params[field_name]['$gt'] = moment.unix(min_max[0]).toDate();
+      if (min_max[0]) condition['$gt'] = min_max[0];
-      if (min_max[1]) params[field_name]['$lt'] = moment.unix(min_max[1]).toDate();
+      if (min_max[1]) condition['$lt'] = min_max[1];
       if (Object.keys(condition).length) params[field_name] = condition;
     }
     return params;

server/lib/tasks/seed_data.js

@@ -18,7 +18,7 @@ export class PowerDataSeed {
       rows = [];
 
     csvStream.on("data", function(data){
-      data.time = moment.utc(parseInt(data.time * 1000)).format();
+      data.time = data.time;
       rows.push(data);
       if (rows.length % 100 === 0){
         DB.PowerDatum.bulkCreate(rows, {validate: true}).catch((error)=>{
@@ -47,13 +47,13 @@ export class PowerDataSeed {
 
   static generateCsv(opts, done){
     opts = extend({
-      start_date: moment().subtract(2, "months").unix(),
+      start_date: moment().subtract(120, "months").unix(),
       end_date: moment().unix(),
       interval: 900, // every 15 minutes (in s)
       average: 1400, // Wh
       path: DATA_PATH + "power_data.csv"
     }, opts || {});
-
+console.log(opts.start_date, opts.end_date)
     var row_date = opts.start_date,
       csvStream = csv.format({headers: true}),
       writableStream = fs.createWriteStream(opts.path),

server/models/energy_datum.js

@@ -15,7 +15,7 @@ var EnergyDatum = DB.sequelize.define(NAME, {
     autoIncrement: true // Automatically gets converted to SERIAL for postgres
   },
   day: {
-    type: DB.Sequelize.DATEONLY,
+    type: DB.Sequelize.INTEGER,
   },
   production: DB.Sequelize.FLOAT,
   consumption: DB.Sequelize.FLOAT
@@ -23,14 +23,7 @@ var EnergyDatum = DB.sequelize.define(NAME, {
   paranoid: true,
   underscored: true,
   tableName: "energy_data",
-  instanceMethods: {
+  instanceMethods: {},
-    exposeToApi: function(){
-      var energy_datum = this,
-        values = energy_datum.dataValues;
-      values.energy_datum = energy_datum.day.getTime() / 1000;
-      return values;
-    }
-  },
   classMethods: {
     set: ()=>{
       EnergyDatum.associate();
@@ -46,7 +39,7 @@ var EnergyDatum = DB.sequelize.define(NAME, {
         attributes: ['id', 'production', 'consumption', 'day']
       }).then((energy_data)=>{
         return energy_data.map((energy_datum)=>{
-          return energy_datum.exposeToApi();
+          return energy_datum.dataValues;
         });
       });
     }

server/models/house.js

@@ -14,7 +14,13 @@ var House = DB.sequelize.define(NAME, {
     autoIncrement: true // Automatically gets converted to SERIAL for postgres
   },
   timezone: DB.Sequelize.STRING,
-  name: DB.Sequelize.STRING
+  name: DB.Sequelize.STRING,
+  data_until: {
+    type: DB.Sequelize.INTEGER,
+  },
+  data_from: {
+    type: DB.Sequelize.INTEGER,
+  }
 }, {
   paranoid: true,
   underscored: true,
@@ -29,26 +35,47 @@ var House = DB.sequelize.define(NAME, {
       }
       return multiplier;
     },
-    timeToDateString: function(timestamp){
+    unixToLocalDay: function(unix){
       var house = this;
-      return moment.tz(timestamp, house.timezone).format("YYYY-MM-DD");
+      return moment.tz(unix * 1000, house.timezone).startOf('day').unix();
     },
     aggregatePowerToEnergyData: function(){
       var house = this;
       return DB.EnergyDatum.destroy({where: {house_id: house.id}})
         .then(()=>{
-          return house.getPowerData();
+          return DB.PowerDatum.count({where: {house_id: house.id}})
         })
-        .then((power_data)=>{
+        .then((count)=>{
-          var energy_data = new Map();
+          var limit = 0,
-          power_data.forEach((power_datum)=>{
+            energy_data = new Map(),
-            var day = house.timeToDateString(power_datum.time),
+            promises = [];
-              energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
+          while (limit < count){
-            energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
+            let complete = DB.PowerDatum.findAll({where: {house_id: house.id}, limit: 1000, offset: limit, order: 'id ASC'})
-            energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
+                    .then((power_data)=>{
-            energy_data.set(day, energy_datum);
+                      power_data.forEach((power_datum)=>{
+                        var day = house.unixToLocalDay(power_datum.time),
+                          energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
+                        energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
+                        energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
+                        energy_data.set(day, energy_datum);
+                      });
+                    });
+            promises.push(complete);
+            limit += 1000;
+          }
+          return Promise.all(promises).then(()=>{
+            return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true});
           });
-          return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true});
+        })
+        .then(()=>{
+          return house.getPowerData({order: 'time DESC', limit: 1});
+        })
+        .then((max_data)=>{
+          house.data_until = max_data[0].time;
+          return house.getPowerData({order: 'time ASC', limit: 1});
+        }).then((min_data)=>{
+          house.data_from = min_data[0].time;
+          return house.save();
         });
     }
   },
@@ -58,16 +85,6 @@ var House = DB.sequelize.define(NAME, {
     },
     associate: ()=>{
       House.hasMany(DB.PowerDatum, {as: 'PowerData'});
-    },
-    getPowerDataByTime: (dates)=>{
-      var params = {
-        where: {time: {}},
-        attributes: ['id', 'time', 'consumption', 'production']
-      };
-      if (start_date) params.where.time.$gt = moment.utc(start_date).toDate();
-      if (end_date) params.where.time.$lt = moment.utc(end_date).toDate();
-
-      return House.getPowerData(params);
     }
   }
 });

server/models/power_datum.js

@@ -15,7 +15,7 @@ var PowerDatum = DB.sequelize.define(NAME, {
     autoIncrement: true // Automatically gets converted to SERIAL for postgres
   },
   time: {
-    type: DB.Sequelize.DATE,
+    type: DB.Sequelize.INTEGER,
   },
   consumption: DB.Sequelize.FLOAT,
   production: DB.Sequelize.FLOAT

shared/utils/date_range.js

@@ -6,9 +6,6 @@ class DateRange {
     if (start === undefined && end === undefined && ranges.length === 0){
       gaps_filled = [undefined, undefined];
       new_ranges = [[undefined, undefined]];
-    } else if (ranges.length === 0){
-      gaps_filled = [new_range];
-      new_ranges = [new_range]
     } else {
       var covered = false,
         last_start = start,
@@ -16,22 +13,22 @@ class DateRange {
 
       ranges.forEach((range, i)=>{
         if (covered){ new_ranges.push(range); return true; }
-        if (DateUtil.lte(start, range[0])){
+        if (DateRange.lte(start, range[0])){
-          if (end && !DateUtil.eq(end, range[0]) && DateUtil.lte(end, range[0])){
+          if (end && !DateRange.eq(end, range[0]) && DateRange.lte(end, range[0])){
             new_ranges.push([last_start, end]);
             new_ranges.push(range);
             gaps_filled.push([last_end, end]);
             covered = true;
-          } else if (end && !DateUtil.gte(end, range[1])) {
+          } else if (end && !DateRange.gte(end, range[1])) {
             new_ranges.push([last_start, range[1]]);
-            if (range[0] && !DateUtil.eq(last_end, range[0])){ gaps_filled.push([last_end, range[0]]); }
+            if (range[0] && !DateRange.eq(last_end, range[0])){ gaps_filled.push([last_end, range[0]]); }
             covered = true
           } else {
-            if (range[0] && !DateUtil.eq(last_end, range[0])) gaps_filled.push([last_end, range[0]]);
+            if (range[0] && !DateRange.eq(last_end, range[0])) gaps_filled.push([last_end, range[0]]);
             last_end = range[1]
           }
-        } else if (start && DateUtil.gte(range[1], start)){
+        } else if (start && DateRange.gte(range[1], start)){
-            if (!DateUtil.eq(end, range[1]) && DateUtil.gte(end, range[1])){
+            if (!DateRange.eq(end, range[1]) && DateRange.gte(end, range[1])){
               last_start = range[0];
               last_end = range[1];
             } else {
@@ -42,7 +39,7 @@ class DateRange {
       });
       if (!covered) {
         new_ranges.push([last_start, end]);
-        if (!DateUtil.eq(last_end, end)) gaps_filled.push([last_end, end]);
+        if (!DateRange.eq(last_end, end)) gaps_filled.push([last_end, end]);
       }
     }
 
@@ -58,7 +55,9 @@ class DateRange {
   }
 
   static eq(date1, date2){
-    return (date1 !== undefined && date2 !== undefined && date1.getTime() === date2.getTime()) || date1 === undefined && date2 === undefined
+    if (date1 && date1.constructor === Date) date1 = date1.getTime();
+    if (date2 && date2.constructor === Date) date2 = date2.getTime();
+    return (date1 !== undefined && date2 !== undefined && date1 === date2) || date1 === undefined && date2 === undefined
   }
 
   static add(date, s){
@@ -66,4 +65,4 @@ class DateRange {
   }
 
 }
-export default DateUtil;
+export default DateRange;