diff --git a/client/api/energy_data.js b/client/api/energy_data.js index 4da4cd5..766808c 100644 --- a/client/api/energy_data.js +++ b/client/api/energy_data.js @@ -5,7 +5,7 @@ const ENDPOINT = '/data/v1/energy'; // send all date parameters as unix timestamps; class EnergyDataApi { - static index(params){ + static index(house, params){ params = extend({ }, params); if (params.dates){ diff --git a/client/api/houses.js b/client/api/houses.js index ed92db1..eb165af 100644 --- a/client/api/houses.js +++ b/client/api/houses.js @@ -1,6 +1,7 @@ -const ENDPOINT = '/data/v1/houses'; import extend from 'extend'; +const ENDPOINT = '/data/v1/houses'; + class HousesApi { static index(params){ diff --git a/client/api/power_data.js b/client/api/power_data.js index f2aea90..d8df20e 100644 --- a/client/api/power_data.js +++ b/client/api/power_data.js @@ -5,15 +5,6 @@ import extend from 'extend'; class PowerDataApi { static index(params){ - params = extend({ - }, params); - if (params.dates){ - params.dates = params.dates.map((date_range)=>{ - if (date_range[0]) date_range[0] = date_range[0].unix(); - if (date_range[1]) date_range[1] = date_range[1].unix(); - return [date_range[0], date_range[1]]; - }) - } return jQuery.ajax({ url: ENDPOINT + '?' + jQuery.param(params), type: 'GET', diff --git a/client/d3/line/spline_stack.js b/client/d3/line/spline_stack.js index 222fde1..f76bbf1 100644 --- a/client/d3/line/spline_stack.js +++ b/client/d3/line/spline_stack.js @@ -30,10 +30,8 @@ class SplineStackChart extends LineChart { var spline_stack = this, serialized_data = { series: [] }; - data.series.forEach(function(series, i){ - series.css_class = series.css_class || spline_stack.toClass ? spline_stack.toClass(series) : ""; - series.title = series.title || spline_stack.toClass ? spline_stack.titleize(series) : ""; + series.css_class = series.css_class || spline_stack.toClass ? spline_stack.toClass(series) : "series-" + i; if (spline_stack.domain_attr !== 'x' && spline_stack.range_attr !== 'y'){ series.values = series.values.map((value)=>{ return {x: value[spline_stack.domain_attr], y: value[spline_stack.range_attr], series: series}; @@ -41,6 +39,7 @@ class SplineStackChart extends LineChart { } serialized_data.series.push(series); }); + serialized_data.series = spline_stack.fnStack(serialized_data.series); // assume all series have same domain, use first series to establish extent. serialized_data.domain_extent = d3.extent(serialized_data.series[0].values.map((value)=>{ return value.x; })); @@ -68,6 +67,17 @@ class SplineStackChart extends LineChart { spline_stack.applyData(paths); }); stack.exit().remove(); + + if (spline_stack.include_dots){ + data.series.forEach((series)=>{ + var dots = spline_stack.svg.selectAll(".d3-chart-spline-dot." + series.css_class) + .data(series.values); + [dots.enter().append("circle"), dots.transition()].forEach((circles)=>{ + spline_stack.applyDots(series, circles); + }); + dots.exit().remove(); + }); + } } applyData(paths){ @@ -75,8 +85,24 @@ class SplineStackChart extends LineChart { paths .attr("class", function(series){"d3-chart-spline-stack " + series.css_class;}) .attr("d", function(series){ return spline_stack.fnArea(series.values); }) - .style("fill", function(series){ return spline_stack.fnColor(series.title); }); + .style("fill", function(series){ return spline_stack.fnColor(series.title); }) + .attr('opacity', 1); } + + applyDots(series, circles){ + var spline_stack = this; + circles + .attr('class', 'd3-chart-spline-dot' + series.css_class) + .attr("r", 2) + .attr("cx", function(d, i){ return spline_stack.x_scale(d.x); }) + .attr("cy", function(d, i){ return spline_stack.y_scale(d.y + d.y0); }) + .attr("title", function(d, i){ return spline_stack.titleizeDatum(series, d); }) + .style("fill", spline_stack.fnColor(series.title)) + .attr('opacity', 1) + .attr('stroke-width', 0) + .attr('stroke', '#fff'); + } + } export default SplineStackChart; diff --git a/client/d3/sliders/date_range.js b/client/d3/sliders/date_range.js new file mode 100644 index 0000000..6e38231 --- /dev/null +++ b/client/d3/sliders/date_range.js @@ -0,0 +1,133 @@ +import Chart from './../base'; + +class DateRange extends Chart { + + + get chart_options(){ + return Object.assign(Object.assign({}, Chart.DEFAULTS), { + outer_width: 600, + outer_height: 250, + margin: {top: 20, left: 10, bottom: 20, right: 10}, + }); + } + + defineAxes(){ + var date_range = this; + + date_range.x_scale = d3.time.scale() + .range([0, date_range.width]) + .clamp(true); + + date_range.x_axis = d3.svg.axis() + .scale(date_range.x_scale) + .orient("bottom") + .ticks(d3.time.weeks, 1) + //.tickFormat(function(d) { return d + "°"; }) + .tickSize(1) + .outerTickSize(1) + .tickPadding(12) + + date_range.svg.append("g") + .attr("class", "d3-chart-domain") + .attr("transform", "translate(0," + date_range.height / 2 + ")"); + } + + afterAxes(){ + var date_range = this; + + date_range.slider = date_range.svg.append("g") + .attr("class", "d3-chart-slider"); + + date_range.min_handle = date_range.slider.append("circle") + .attr("class", "d3-chart-min-handle") + .attr("transform", "translate(0," + date_range.height / 2 + ")") + .attr("r", 9); + + date_range.max_handle = date_range.slider.append("circle") + .attr("class", "d3-chart-max-handle") + .attr("transform", "translate(0," + date_range.height / 2 + ")") + .attr("r", 9); + + date_range.brush = d3.svg.brush() + .x(date_range.x_scale); + + date_range.slider + .call(date_range.brush) + //.select(".background") + // .attr("height", date_range.height); + + date_range.slider.call(date_range.brush) + .selectAll(".extent,.resize") + .remove(); + } + + drawData(data){ + var date_range = this; + date_range.x_scale.domain([data.abs_min, data.abs_max]); + + date_range.svg.select(".d3-chart-domain") + .call(date_range.x_axis); + + date_range.min_handle.attr('cx', date_range.x_scale(data.current_min)); + date_range.max_handle.attr('cx', date_range.x_scale(data.current_max)); + + date_range.brush.extent([data.current_min, data.current_min]) + .on("brush", ()=>{ + DateRange.handleBrush(date_range, eval('this')); + }); + + date_range.slider + .call(date_range.brush.event) + .transition() // gratuitous intro! + .duration(750) + .call(date_range.brush.extent([data.current_min, data.current_min])) + .call(date_range.brush.event); + + } + + static handleBrush(date_range, elem){ + var date = date_range.brush.extent()[0], + current_min = parseInt(date_range.min_handle.attr('cx')), + current_max = parseInt(date_range.max_handle.attr('cx')); + + if (!current_min && !current_max) return false + if (d3.event.sourceEvent) { // not a programmatic event + date = date_range.x_scale.invert(d3.mouse(elem)[0]); + date_range.brush.extent([date, date]); + } + + var value = date_range.x_scale(date); + + if (value < current_max && value > current_min){ + if (Math.abs(value - current_min) < Math.abs(value - current_max)){ + date_range.min_handle.attr('cx', value); + current_min = value; + } else { + date_range.max_handle.attr('cx', value); + current_max = value; + } + } else if (value >= current_max){ + date_range.max_handle.attr('cx', value); + current_max = value; + if (d3.event.sourceEvent && date_range.maxDelta){ + var new_current_min = date_range.maxDelta(date, date_range.x_scale.invert(current_min)); + if (new_current_min) date_range.min_handle.attr('cx', date_range.x_scale(new_current_min)); + } + } else { + date_range.min_handle.attr('cx', value); + current_min = value; + if (d3.event.sourceEvent && date_range.maxDelta){ + var new_current_max = date_range.maxDelta(date, date_range.x_scale.invert(current_max)); + if (new_current_max) date_range.max_handle.attr('cx', date_range.x_scale(new_current_max)); + } + } + + + if (d3.event.sourceEvent && date_range.onRangeUpdated) { + date_range.onRangeUpdated(date_range.x_scale.invert(current_min), date_range.x_scale.invert(current_max)); + } + } + +} + +export default DateRange; diff --git a/client/dashboard/energy/energy.js b/client/dashboard/energy/energy.js index 3329254..d06df2d 100644 --- a/client/dashboard/energy/energy.js +++ b/client/dashboard/energy/energy.js @@ -22,7 +22,7 @@ var Energy = React.createClass({ var energy = this, house = energy.props.house; energy.graph_title = 'Daily Consumption'; - house.ensureEnergyData().then(()=>{ + house.setEnergyData().then(()=>{ energy.setState({loading_data: false}); if (energy.props.view === 'graph') energy.initGraph(); }); @@ -32,7 +32,7 @@ var Energy = React.createClass({ var energy = this; if (new_props.house !== energy.state.house){ energy.setState({loading_data: true}); - new_props.house.ensureEnergyData().then(()=>{ + new_props.house.setEnergyData().then(()=>{ energy.setState({loading_data: false}); if (energy.props.view === 'graph') energy.initGraph(); }); diff --git a/client/dashboard/energy/energy.rt b/client/dashboard/energy/energy.rt index 8b7cea3..4c94242 100644 --- a/client/dashboard/energy/energy.rt +++ b/client/dashboard/energy/energy.rt @@ -27,7 +27,7 @@ - + {energy_datum.day_to_s} {energy_datum.consumption_to_s} diff --git a/client/dashboard/energy/energy.rt.js b/client/dashboard/energy/energy.rt.js index cb2b4a4..a17cbd5 100644 --- a/client/dashboard/energy/energy.rt.js +++ b/client/dashboard/energy/energy.rt.js @@ -1,7 +1,7 @@ import React from 'react'; import _ from 'lodash'; function repeatEnergy_datum1(energy_datum, energy_datumIndex) { - return React.createElement('tr', { 'key': energy_datum.react_key }, React.createElement('td', {}), React.createElement('td', {}, energy_datum.day_to_s), React.createElement('td', {}, energy_datum.consumption_to_s), React.createElement('td', {}, energy_datum.production_to_s)); + return React.createElement('tr', { 'key': energy_datum.scoped_id }, React.createElement('td', {}), React.createElement('td', {}, energy_datum.day_to_s), React.createElement('td', {}, energy_datum.consumption_to_s), React.createElement('td', {}, energy_datum.production_to_s)); } export default function () { return React.createElement('div', { 'id': 'energy_view' }, this.state.loading_data ? React.createElement('div', { 'className': 'alert alert-warning' }, '\n Retrieving energy data for the ', this.props.house.name, ' household...\n ') : null, this.props.view === 'graph' ? React.createElement('div', {}, React.createElement('h4', {}, 'Select Data'), React.createElement('div', { diff --git a/client/dashboard/layout/layout.js b/client/dashboard/layout/layout.js index f945bd4..aeaca3e 100644 --- a/client/dashboard/layout/layout.js +++ b/client/dashboard/layout/layout.js @@ -1,6 +1,7 @@ import React from 'react'; import layoutRt from './layout.rt.js'; import House from './../../models/house'; +import PowerDatum from './../../models/power_datum'; var Layout = React.createClass({ @@ -31,7 +32,9 @@ var Layout = React.createClass({ var layout = this, house_id = event.target.value, house = layout.state.houses.find((house)=>{ return house.data.id == house_id }); - layout.setState({house: house}); + layout.setState({house: house}, ()=>{ + house.closeDb(); + }); }, setView: function(event){ @@ -47,6 +50,19 @@ var Layout = React.createClass({ layout.setState({dataset: dataset}); }, + refreshData: function(){ + var layout = this, + houses = layout.state.houses, + all = []; + houses.forEach((house)=>{ + all.push(house.clearData()); + }); + Promise.all(all) + .then(()=>{ + window.location.reload(); + }); + }, + render: function() { return layoutRt.call(this); } diff --git a/client/dashboard/layout/layout.rt b/client/dashboard/layout/layout.rt index 62768b6..229cfc7 100644 --- a/client/dashboard/layout/layout.rt +++ b/client/dashboard/layout/layout.rt @@ -2,12 +2,12 @@
Retrieving houses...
-

{this.state.house.name}

Select household:

+

Select dataset:

diff --git a/client/dashboard/layout/layout.rt.js b/client/dashboard/layout/layout.rt.js index ef830c5..6230013 100644 --- a/client/dashboard/layout/layout.rt.js +++ b/client/dashboard/layout/layout.rt.js @@ -5,18 +5,21 @@ import Power from './../power/power'; function repeatHouse1(house, houseIndex) { return React.createElement('option', { 'value': house.data.id, - 'key': house.react_key + 'key': house.scoped_id }, house.data.name); } export default function () { - return React.createElement('div', { 'id': 'layout' }, this.state.requesting_data ? React.createElement('div', { 'className': 'alert alert-warning' }, 'Retrieving houses...') : null, this.state.house ? React.createElement('h1', {}, this.state.house.name) : null, React.createElement('h4', {}, 'Select household:'), this.state.houses ? React.createElement.apply(this, [ + return React.createElement('div', { 'id': 'layout' }, this.state.requesting_data ? React.createElement('div', { 'className': 'alert alert-warning' }, 'Retrieving houses...') : null, React.createElement('h4', {}, 'Select household:'), this.state.houses ? React.createElement.apply(this, [ 'select', { 'className': 'form-control', 'onChange': this.setHouse }, _.map(this.state.houses, repeatHouse1.bind(this)) - ]) : null, React.createElement('h4', {}, 'Select dataset:'), React.createElement('div', { + ]) : null, this.state.house ? React.createElement('button', { + 'onClick': this.refreshData, + 'className': 'btn btn-xs btn-default' + }, 'Refresh House Data') : null, React.createElement('h4', {}, 'Select dataset:'), React.createElement('div', { 'className': 'btn-group', 'role': 'group' }, React.createElement('button', { diff --git a/client/dashboard/power/power.js b/client/dashboard/power/power.js index 7f133e5..ed49eba 100644 --- a/client/dashboard/power/power.js +++ b/client/dashboard/power/power.js @@ -1,7 +1,11 @@ import React from 'react'; +import moment from 'moment-timezone'; +import _ from 'lodash'; + import powerRt from './power.rt.js'; import House from './../../models/house'; import SplineStackChart from './../../d3/line/spline_stack'; +import DateRangeSlider from './../../d3/sliders/date_range'; var Power = React.createClass({ @@ -21,45 +25,63 @@ var Power = React.createClass({ var power = this, house = power.props.house; power.graph_title = ''; - house.ensurePowerData().then(()=>{ + power.initDateRange(); + house.setPowerData().then(()=>{ power.setState({loading_data: false}); - if (power.props.view === 'graph') power.initGraph(); + if (power.props.view === 'graph'){ + power.initGraph(); + } }); }, componentWillReceiveProps: function(new_props){ var power = this; if (new_props.house !== power.state.house){ + // house will change. power.setState({loading_data: true}); - new_props.house.ensurePowerData().then(()=>{ + new_props.house.setPowerData().then(()=>{ power.setState({loading_data: false}); - if (power.props.view === 'graph') power.initGraph(); + if (power.props.view === 'graph'){ + power.initDateRange(); + power.initGraph(); + } }); } + // view will change from graph to table. if (new_props.view !== 'graph' && power.props.view === 'graph') power.destroyGraph(); }, componentDidUpdate: function(prev_props, _prev_state){ var power = this, house = power.props.house; - if (prev_props.view !== 'graph' && power.props.view === 'graph') power.initGraph(); + // view has changed from graph to table. + if (prev_props.view !== 'graph' && power.props.view === 'graph'){ + power.initDateRange(); + power.initGraph(); + } }, initGraph: function(){ - var power = this; + var power = this, + house = power.props.house; if (power.graph === undefined){ power.graph = new SplineStackChart({ container: '#power_graph', outer_width: 800, outer_height: 200, color: '#0404B4', - range_attr: 'y', + time_series: true, domain_attr: 'x', - time_series: true + range_attr: 'y', + include_dots: true, + titleizeDatum: (series, d)=>{ + return series.title + '
' + Math.round(d.y) + ' W
' + moment.tz(d.x.getTime(), house.timezone).format('MMM D [at] HH:mm') + } }); jQuery('#power_graph').tooltip({ - selector: 'path', + selector: 'circle', container: 'body', + html: true, title: function(){ return this.__data__.title; } @@ -73,11 +95,19 @@ var Power = React.createClass({ house = power.props.house, net_power = { title: 'Net Power Consumption', - values: house.power_data.slice(0, 200).map((power_datum)=>{ return {y: Math.max(0, power_datum.data.consumption - power_datum.data.production), x: power_datum.data.time.toDate() } }) + values: house.power_data.map((power_datum)=>{ + return { + x: house.toDate(power_datum.data.time), + y: Math.max(0, power_datum.data.consumption - power_datum.data.production) } + }) }, savings = { title: 'Power Production', - values: house.power_data.slice(0, 200).map((power_datum)=>{ return {y: power_datum.data.production, x: power_datum.data.time.toDate() } }) + values: house.power_data.map((power_datum)=>{ + return { + x: house.toDate(power_datum.data.time), + y: power_datum.data.production } + }) }; power.graph.drawData({ title: power.graph_title, @@ -86,9 +116,50 @@ var Power = React.createClass({ }); }, + initDateRange: function(){ + var power = this, + house = power.props.house; + if (power.date_range_slider === undefined){ + power.date_range_slider = new DateRangeSlider({ + container: '#power_date_setter', + outer_height: 100, + maxDelta: function(changed_date, other_date){ + if (Math.abs(changed_date.getTime() - other_date.getTime()) > 3600 * 24 * 7 * 1000){ + if (changed_date > other_date){ + return new Date(changed_date.getTime() - 3600 * 24 * 4 * 1000); + } else { + return new Date(changed_date.getTime() + 3600 * 24 * 4 * 1000); + } + } + return false; + } + }); + } + power.date_range_slider.onRangeUpdated = (min, max)=>{ + if (power.date_range_update) clearTimeout(power.date_range_update) + power.date_range_update = setTimeout(()=>{ + house.power_date_range = [Math.round(min.getTime() / 1000), Math.round(max.getTime() / 1000)] + house.setPowerData() + .then(()=>{ + power.updateGraph(); + }); + }, 500); + }; + var four_week_start = house.data.data_until - 3600 * 24 * 28, + data_from = Math.max(four_week_start, house.data.data_from); + power.date_range_slider.drawData({ + abs_min: house.toDate(four_week_start), + abs_max: house.toDate(house.data.data_until), + current_min: house.toDate(house.default_power_start), + current_max: house.toDate(house.default_power_end) + }); + }, + destroyGraph: function(){ var power = this; + document.getElementById('power_date_setter').innerHTML = ''; document.getElementById('power_graph').innerHTML = ''; + power.date_range_slider = undefined; power.graph = undefined; }, diff --git a/client/dashboard/power/power.rt b/client/dashboard/power/power.rt index 4cebd67..9d27f81 100644 --- a/client/dashboard/power/power.rt +++ b/client/dashboard/power/power.rt @@ -1,8 +1,9 @@
-

Household 15-minute Power Statistics

Retrieving power data for the {this.props.house.name} household...
+

Select dates:

+
@@ -13,7 +14,7 @@ - + diff --git a/client/dashboard/power/power.rt.js b/client/dashboard/power/power.rt.js index b455bce..d349a29 100644 --- a/client/dashboard/power/power.rt.js +++ b/client/dashboard/power/power.rt.js @@ -1,10 +1,10 @@ import React from 'react'; import _ from 'lodash'; function repeatPower_datum1(power_datum, power_datumIndex) { - return React.createElement('tr', { 'key': power_datum.react_key }, React.createElement('td', {}), React.createElement('td', {}, power_datum.time_to_s), React.createElement('td', {}, power_datum.consumption_to_s), React.createElement('td', {}, power_datum.production_to_s)); + return React.createElement('tr', { 'key': power_datum.scoped_id }, React.createElement('td', {}), React.createElement('td', {}, power_datum.time_to_s), React.createElement('td', {}, power_datum.consumption_to_s), React.createElement('td', {}, power_datum.production_to_s)); } export default function () { - return React.createElement('div', { 'id': 'power_view' }, React.createElement('h2', {}, 'Household 15-minute Power Statistics'), this.state.loading_data ? React.createElement('div', { 'className': 'alert alert-warning' }, '\n Retrieving power data for the ', this.props.house.name, ' household...\n ') : null, this.props.view === 'table' ? React.createElement('table', { 'className': 'table' }, React.createElement('thead', {}, React.createElement('tr', {}, React.createElement('th', {}), React.createElement('th', {}, 'Time'), React.createElement('th', {}, 'Consumption (W)'), React.createElement('th', {}, 'Production (W)'))), React.createElement.apply(this, [ + return React.createElement('div', { 'id': 'power_view' }, this.state.loading_data ? React.createElement('div', { 'className': 'alert alert-warning' }, '\n Retrieving power data for the ', this.props.house.name, ' household...\n ') : null, React.createElement('h4', {}, 'Select dates:'), React.createElement('div', { 'id': 'power_date_setter' }), this.props.view === 'table' ? React.createElement('table', { 'className': 'table' }, React.createElement('thead', {}, React.createElement('tr', {}, React.createElement('th', {}), React.createElement('th', {}, 'Time'), React.createElement('th', {}, 'Consumption (W)'), React.createElement('th', {}, 'Production (W)'))), React.createElement.apply(this, [ 'tbody', {}, _.map(this.props.house.power_data, repeatPower_datum1.bind(this)) diff --git a/client/lib/databasable.js b/client/lib/databasable.js index d42932f..9d50e32 100644 --- a/client/lib/databasable.js +++ b/client/lib/databasable.js @@ -1,5 +1,5 @@ -import Loki from 'lokijs'; -import LokiIndexedAdapter from 'loki/loki-indexed-adapter'; +import Loki from 'lokijs/src/lokijs'; +import LokiIndexedAdapter from 'lokijs/src/loki-indexed-adapter'; const DEFAULTS = { autosave: false @@ -10,16 +10,16 @@ var databasable = { accessDb: function(db_name, opts){ var databasable = this; opts = Object.assign(Object.assign({ - persistenceMethod: 'adapter', - persistenceAdapter: new LokiIndexedAdapter(db_name) + adapter: new LokiIndexedAdapter(db_name) }, DEFAULTS), opts || {}); - return new Promise((fnResolve, fnReject){ + return new Promise((fnResolve, fnReject)=>{ if (!databasable.db) { databasable.db = new Loki(db_name, opts); databasable.db.loadDatabase({}, ()=>{ fnResolve(databasable.db); }); } else { fnResolve(databasable.db); } + }); }, @@ -35,14 +35,19 @@ var databasable = { collection: function(collection_name, options){ var databasable = this; return databasable.accessDb() - .then((db)=>{ - var collection = db.getCollection(collection_name) - if (!collection){ - collection = db.addCollection(collection_name, options); - } - return collection; - }); - } + .then((db)=>{ + var collection = db.getCollection(collection_name) + if (!collection){ + collection = db.addCollection(collection_name, options); + } + if (options && options.unique_indices){ + options.unique_indices.forEach((field)=>{ + collection.ensureUniqueIndex(field); + }); + } + return collection; + }); + }, rangeToLokiParams: function(attr, range){ var date_params = {}; @@ -50,7 +55,7 @@ var databasable = { var start_condition = {}, end_condition = {}; date_params['$and'] = [start_condition, end_condition]; - start_condition[attr] = {'$lt': range[1]}; + start_condition[attr] = {'$gt': range[0]}; end_condition[attr] = {'$lt': range[1]}; } else if (range[0] !== undefined) { date_params[attr] = {'$gt': range[0]} @@ -62,4 +67,4 @@ var databasable = { }; -export default databaseable; +export default databasable; diff --git a/client/models/energy_datum.js b/client/models/energy_datum.js index 5347cf1..d92f062 100644 --- a/client/models/energy_datum.js +++ b/client/models/energy_datum.js @@ -12,7 +12,6 @@ class EnergyDatum { energy_datum.house = house; data.day = moment.tz(data.day, house.data.timezone); energy_datum.data = data; - EnergyDatum.store.set(data.id, energy_datum); } get react_key(){ diff --git a/client/models/house.js b/client/models/house.js index e9855ff..2b27dc2 100644 --- a/client/models/house.js +++ b/client/models/house.js @@ -1,5 +1,6 @@ import extend from 'extend'; -import Loki from 'loki'; +import Loki from 'lokijs/src/lokijs'; +import moment from 'moment-timezone'; import PowerDatum from './power_datum'; import EnergyDatum from './energy_datum'; @@ -18,30 +19,57 @@ class House { var house = this; house.data = data; Object.assign(house, Databasable); + house.power_date_range = [house.default_power_start, house.default_power_end]; } get scoped_id(){ return `house-${this.data.id}`; } + get default_power_start(){ + var house = this; + // 3600 * 24 seconds * 4 = 4 days. + return house.data.data_until - 3600 * 24 * 4; + } + + get default_power_end(){ + var house = this; + return house.data.data_until; + } + + toDate(unix){ + var house = this; + return moment.tz(unix * 1000, house.data.timezone).toDate(); + } + save(){ var house = this; - return House.collection() + return House.collection(House.NAME) .then((house_collection)=>{ - return house_collection.update(house.data); + house_collection.update(house.data); + return House.db.save(); }); } setPowerData(opts){ var house = this; - return house.ensurePowerData(opts) - .then(()=>{ - return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS) - .then((power_collection)=>{ - var params = house.rangeToLokiParams('time', [opts.start_date, opts.end_date]); - house.power_data = power_collection.find(params).map((data)=>{ return new PowerDatum(data); }) + opts = Object.assign({ + dates: house.power_date_range + }, opts || {}); + return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS) + .then((power_collection)=>{ + return house.ensurePowerData(opts) + .then(()=>{ + var params = house.rangeToLokiParams('time', opts.dates); + house.power_data = power_collection.find(params) + .sort((pd1, pd2)=>{ + if (pd1.time === pd2.time) return 0; + if (pd1.time > pd2.time) return 1; + if (pd1.time < pd2.time) return -1; + }) + .map((data)=>{ return new PowerDatum(data, house); }) }); - }) + }); } ensurePowerData(opts){ @@ -50,13 +78,16 @@ class House { end_date: undefined }, opts || {}); var house = this, - query_ranges = DateRange.addRange([opts.start_date, opts.end_date], house.data.power_datum_ranges); + existing_ranges = house.data.power_datum_ranges || [], + query_ranges; + query_ranges = DateRange.addRange(opts.dates, existing_ranges); if (query_ranges.gaps_filled.length > 0){ - house.getPowerData({dates: query_ranges.gaps_filled}) + var params = {dates: query_ranges.gaps_filled}; + return house.getPowerData(params) .then(()=>{ house.data.power_datum_ranges = query_ranges.new_ranges; - return house.save(); + house.save(); }); } else { return Promise.resolve(); } } @@ -66,14 +97,33 @@ class House { params.house_id = house.data.id; return PowerDataApi.index(params) .then((power_data)=>{ - return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS); + return house.collection(PowerDatum.NAME, PowerDatum.COLLECTION_OPTIONS) .then((power_collection)=>{ power_collection.insert(power_data); - return house.db.save(); + house.db.save(); }); }) } + clearData(){ + var house = this; + return new Promise((fnResolve, fnReject)=>{ + house.collection(PowerDatum.NAME) + .then((power_collection)=>{ + power_collection.removeWhere({}); + house.db.save(()=>{ + House.collection(House.NAME) + .then((house_collection)=>{ + house_collection.remove(house.data); + House.db.save(()=>{ + fnResolve(); + }) + }); + }); + }); + }); + } + setEnergyData(opts){ var house = this; return house.ensureEnergyData(opts) @@ -81,7 +131,7 @@ class House { return house.collection(EnergyDatum.NAME, EnergyDatum.COLLECTION_OPTIONS) .then((energy_collection)=>{ var params = house.rangeToLokiParams('day', [opts.start_date, opts.end_date]); - house.energy_data = energy_collection.find(params).map((data)=>{ return new EnergyDatum(data); }) + house.energy_data = energy_collection.find(params).map((data)=>{ return new EnergyDatum(data, house); }) }); }) } @@ -108,7 +158,7 @@ class House { params.house_id = house.data.id; return EnergyDataApi.index(params) .then((energy_data)=>{ - return house.collection(EnergyDatum.NAME, EnergyDatum.COLLECTION_OPTIONS); + return house.collection(EnergyDatum.NAME, EnergyDatum.COLLECTION_OPTIONS) .then((energy_collection)=>{ energy_collection.insert(energy_data); return house.db.save(); @@ -117,34 +167,25 @@ class House { } static ensureHouses(ids){ - House.collection() + return House.collection(House.NAME) .then((house_collection)=>{ - houses = house_collection.find({id: {$in: ids}}); - if (houses.length !== ids.length){ - required_ids = ArrayUtil.diff(ids, houses.map((house)=>{ return house.id; })); - return House.getHouses(required_ids) - .then((required_houses){ - return houses.concat(required_houses); + var houses_data = ids ? house_collection.find({id: {$in: ids}}) : house_collection.find(); + if (!ids && houses_data.length === 0 || ids && houses_data.length !== ids.length){ + var required_ids = ids ? ArrayUtil.diff(ids, houses_data.map((data)=>{ return data.id; })) : undefined; + return HousesApi.index({id: ids}) + .then((required_houses)=>{ + required_houses.forEach((house_data)=>{ + house_collection.insert(house_data); + }); + House.db.save(); + return houses_data.concat(required_houses); }); - } else { return houses; } - }).then((house_data)=>{ + } else { return Promise.resolve(houses_data); } + }).then((houses_data)=>{ return houses_data.map((house_data)=>{ return new House(house_data); }) }); } - static getHouses(ids){ - return HousesApi.index({id: ids}) - .then((houses_data)=>{ - return House.collection() - .then((house_collection)=>{ - houses_data.forEach((house_data)=>{ - house_collection.insert(house_data); - }); - return houses_data; - }); - }); - } - } Object.assign(House, Databasable); diff --git a/client/models/power_datum.js b/client/models/power_datum.js index f7e1668..0b90ed9 100644 --- a/client/models/power_datum.js +++ b/client/models/power_datum.js @@ -3,16 +3,15 @@ import moment from 'moment-timezone'; const NAME = 'PowerDatum'; const COLLECTION_DEFAULTS = { - indices: ['time'] + indices: ['time'], + unique_indices: ['time'] }; class PowerDatum { constructor(data, house){ var power_datum = this; power_datum.house = house; - data.time = moment.tz(data.time, house.data.timezone); power_datum.data = data; - PowerDatum.store.set(data.id, power_datum); } get react_key(){ diff --git a/server/config/webpack/development.js b/server/config/webpack/development.js index aa8502b..897de9e 100644 --- a/server/config/webpack/development.js +++ b/server/config/webpack/development.js @@ -41,5 +41,8 @@ module.exports = { d3: "d3", "window.d3": "d3" }) - ] + ], + node: { + fs: "empty" + } } diff --git a/server/config/webpack/production.js b/server/config/webpack/production.js index d4aad85..dae28da 100644 --- a/server/config/webpack/production.js +++ b/server/config/webpack/production.js @@ -51,5 +51,8 @@ module.exports = { jQuery: "jquery", "window.jQuery": "jquery" }) - ] + ], + node: { + fs: "empty" + } }; diff --git a/server/controllers/houses_controller.js b/server/controllers/houses_controller.js index 9b4c328..3fa4d8e 100644 --- a/server/controllers/houses_controller.js +++ b/server/controllers/houses_controller.js @@ -7,7 +7,7 @@ class HousesController { static index(req, res){ var params = {}; if (req.query.ids) query.id = ids; - DB.House.findAll({where: params, attributes: ['id', 'name', 'timezone']}).then((houses)=>{ + DB.House.findAll({where: params}).then((houses)=>{ res.json({data: houses.map((house)=>{ return house.dataValues; })}); }); } diff --git a/server/controllers/power_controller.js b/server/controllers/power_controller.js index 59ca459..8c4654b 100644 --- a/server/controllers/power_controller.js +++ b/server/controllers/power_controller.js @@ -5,7 +5,6 @@ const NAME = 'PowerController'; class PowerController{ static index(req, res){ - console.log(req.query); DB.PowerDatum.exposeForHouseAtDates(req.query.house_id, req.query.dates).then((power_data)=>{ res.json({data: power_data}); }); diff --git a/server/helpers/api_helper.js b/server/helpers/api_helper.js index a62754a..c17f5b6 100644 --- a/server/helpers/api_helper.js +++ b/server/helpers/api_helper.js @@ -1,5 +1,3 @@ -import moment from 'moment'; - class ApiHelper { // assume all dates from api coming as UNIX timestamps. @@ -12,15 +10,15 @@ class ApiHelper { dates.forEach((min_max)=>{ var condition_n = {}; condition_n[field_name] = {}; - if (min_max[0]) condition_n[field_name]['$gt'] = moment.unix(min_max[0]).toDate(); - if (min_max[1]) condition_n[field_name]['$lt'] = moment.unix(min_max[1]).toDate(); + if (min_max[0]) condition_n[field_name]['$gt'] = min_max[0]; + if (min_max[1]) condition_n[field_name]['$lt'] = min_max[1]; if (Object.keys(condition_n).length) params['$or'].push(condition_n); }); } else { var min_max = dates[0], condition = {} - if (min_max[0]) params[field_name]['$gt'] = moment.unix(min_max[0]).toDate(); - if (min_max[1]) params[field_name]['$lt'] = moment.unix(min_max[1]).toDate(); + if (min_max[0]) condition['$gt'] = min_max[0]; + if (min_max[1]) condition['$lt'] = min_max[1]; if (Object.keys(condition).length) params[field_name] = condition; } return params; diff --git a/server/lib/tasks/seed_data.js b/server/lib/tasks/seed_data.js index 797ee5e..1732dd0 100644 --- a/server/lib/tasks/seed_data.js +++ b/server/lib/tasks/seed_data.js @@ -18,7 +18,7 @@ export class PowerDataSeed { rows = []; csvStream.on("data", function(data){ - data.time = moment.utc(parseInt(data.time * 1000)).format(); + data.time = data.time; rows.push(data); if (rows.length % 100 === 0){ DB.PowerDatum.bulkCreate(rows, {validate: true}).catch((error)=>{ @@ -47,13 +47,13 @@ export class PowerDataSeed { static generateCsv(opts, done){ opts = extend({ - start_date: moment().subtract(2, "months").unix(), + start_date: moment().subtract(120, "months").unix(), end_date: moment().unix(), interval: 900, // every 15 minutes (in s) average: 1400, // Wh path: DATA_PATH + "power_data.csv" }, opts || {}); - +console.log(opts.start_date, opts.end_date) var row_date = opts.start_date, csvStream = csv.format({headers: true}), writableStream = fs.createWriteStream(opts.path), diff --git a/server/models/energy_datum.js b/server/models/energy_datum.js index 5d54a71..524da39 100644 --- a/server/models/energy_datum.js +++ b/server/models/energy_datum.js @@ -15,7 +15,7 @@ var EnergyDatum = DB.sequelize.define(NAME, { autoIncrement: true // Automatically gets converted to SERIAL for postgres }, day: { - type: DB.Sequelize.DATEONLY, + type: DB.Sequelize.INTEGER, }, production: DB.Sequelize.FLOAT, consumption: DB.Sequelize.FLOAT @@ -23,14 +23,7 @@ var EnergyDatum = DB.sequelize.define(NAME, { paranoid: true, underscored: true, tableName: "energy_data", - instanceMethods: { - exposeToApi: function(){ - var energy_datum = this, - values = energy_datum.dataValues; - values.energy_datum = energy_datum.day.getTime() / 1000; - return values; - } - }, + instanceMethods: {}, classMethods: { set: ()=>{ EnergyDatum.associate(); @@ -46,7 +39,7 @@ var EnergyDatum = DB.sequelize.define(NAME, { attributes: ['id', 'production', 'consumption', 'day'] }).then((energy_data)=>{ return energy_data.map((energy_datum)=>{ - return energy_datum.exposeToApi(); + return energy_datum.dataValues; }); }); } diff --git a/server/models/house.js b/server/models/house.js index 1cd9e3c..2700180 100644 --- a/server/models/house.js +++ b/server/models/house.js @@ -14,7 +14,13 @@ var House = DB.sequelize.define(NAME, { autoIncrement: true // Automatically gets converted to SERIAL for postgres }, timezone: DB.Sequelize.STRING, - name: DB.Sequelize.STRING + name: DB.Sequelize.STRING, + data_until: { + type: DB.Sequelize.INTEGER, + }, + data_from: { + type: DB.Sequelize.INTEGER, + } }, { paranoid: true, underscored: true, @@ -29,26 +35,47 @@ var House = DB.sequelize.define(NAME, { } return multiplier; }, - timeToDateString: function(timestamp){ + unixToLocalDay: function(unix){ var house = this; - return moment.tz(timestamp, house.timezone).format("YYYY-MM-DD"); + return moment.tz(unix * 1000, house.timezone).startOf('day').unix(); }, aggregatePowerToEnergyData: function(){ var house = this; return DB.EnergyDatum.destroy({where: {house_id: house.id}}) .then(()=>{ - return house.getPowerData(); + return DB.PowerDatum.count({where: {house_id: house.id}}) }) - .then((power_data)=>{ - var energy_data = new Map(); - power_data.forEach((power_datum)=>{ - var day = house.timeToDateString(power_datum.time), - energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id}; - energy_datum.production += power_datum.production / 1000; // convert Wh to kWh - energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh - energy_data.set(day, energy_datum); + .then((count)=>{ + var limit = 0, + energy_data = new Map(), + promises = []; + while (limit < count){ + let complete = DB.PowerDatum.findAll({where: {house_id: house.id}, limit: 1000, offset: limit, order: 'id ASC'}) + .then((power_data)=>{ + power_data.forEach((power_datum)=>{ + var day = house.unixToLocalDay(power_datum.time), + energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id}; + energy_datum.production += power_datum.production / 1000; // convert Wh to kWh + energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh + energy_data.set(day, energy_datum); + }); + }); + promises.push(complete); + limit += 1000; + } + return Promise.all(promises).then(()=>{ + return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true}); }); - return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true}); + }) + .then(()=>{ + return house.getPowerData({order: 'time DESC', limit: 1}); + }) + .then((max_data)=>{ + house.data_until = max_data[0].time; + return house.getPowerData({order: 'time ASC', limit: 1}); + }).then((min_data)=>{ + house.data_from = min_data[0].time; + return house.save(); }); } }, @@ -58,16 +85,6 @@ var House = DB.sequelize.define(NAME, { }, associate: ()=>{ House.hasMany(DB.PowerDatum, {as: 'PowerData'}); - }, - getPowerDataByTime: (dates)=>{ - var params = { - where: {time: {}}, - attributes: ['id', 'time', 'consumption', 'production'] - }; - if (start_date) params.where.time.$gt = moment.utc(start_date).toDate(); - if (end_date) params.where.time.$lt = moment.utc(end_date).toDate(); - - return House.getPowerData(params); } } }); diff --git a/server/models/power_datum.js b/server/models/power_datum.js index 99b29e2..642a073 100644 --- a/server/models/power_datum.js +++ b/server/models/power_datum.js @@ -15,7 +15,7 @@ var PowerDatum = DB.sequelize.define(NAME, { autoIncrement: true // Automatically gets converted to SERIAL for postgres }, time: { - type: DB.Sequelize.DATE, + type: DB.Sequelize.INTEGER, }, consumption: DB.Sequelize.FLOAT, production: DB.Sequelize.FLOAT diff --git a/shared/utils/date_range.js b/shared/utils/date_range.js index ab80e14..04be8e1 100644 --- a/shared/utils/date_range.js +++ b/shared/utils/date_range.js @@ -6,9 +6,6 @@ class DateRange { if (start === undefined && end === undefined && ranges.length === 0){ gaps_filled = [undefined, undefined]; new_ranges = [[undefined, undefined]]; - } else if (ranges.length === 0){ - gaps_filled = [new_range]; - new_ranges = [new_range] } else { var covered = false, last_start = start, @@ -16,22 +13,22 @@ class DateRange { ranges.forEach((range, i)=>{ if (covered){ new_ranges.push(range); return true; } - if (DateUtil.lte(start, range[0])){ - if (end && !DateUtil.eq(end, range[0]) && DateUtil.lte(end, range[0])){ + if (DateRange.lte(start, range[0])){ + if (end && !DateRange.eq(end, range[0]) && DateRange.lte(end, range[0])){ new_ranges.push([last_start, end]); new_ranges.push(range); gaps_filled.push([last_end, end]); covered = true; - } else if (end && !DateUtil.gte(end, range[1])) { + } else if (end && !DateRange.gte(end, range[1])) { new_ranges.push([last_start, range[1]]); - if (range[0] && !DateUtil.eq(last_end, range[0])){ gaps_filled.push([last_end, range[0]]); } + if (range[0] && !DateRange.eq(last_end, range[0])){ gaps_filled.push([last_end, range[0]]); } covered = true } else { - if (range[0] && !DateUtil.eq(last_end, range[0])) gaps_filled.push([last_end, range[0]]); + if (range[0] && !DateRange.eq(last_end, range[0])) gaps_filled.push([last_end, range[0]]); last_end = range[1] } - } else if (start && DateUtil.gte(range[1], start)){ - if (!DateUtil.eq(end, range[1]) && DateUtil.gte(end, range[1])){ + } else if (start && DateRange.gte(range[1], start)){ + if (!DateRange.eq(end, range[1]) && DateRange.gte(end, range[1])){ last_start = range[0]; last_end = range[1]; } else { @@ -42,7 +39,7 @@ class DateRange { }); if (!covered) { new_ranges.push([last_start, end]); - if (!DateUtil.eq(last_end, end)) gaps_filled.push([last_end, end]); + if (!DateRange.eq(last_end, end)) gaps_filled.push([last_end, end]); } } @@ -58,7 +55,9 @@ class DateRange { } static eq(date1, date2){ - return (date1 !== undefined && date2 !== undefined && date1.getTime() === date2.getTime()) || date1 === undefined && date2 === undefined + if (date1 && date1.constructor === Date) date1 = date1.getTime(); + if (date2 && date2.constructor === Date) date2 = date2.getTime(); + return (date1 !== undefined && date2 !== undefined && date1 === date2) || date1 === undefined && date2 === undefined } static add(date, s){ @@ -66,4 +65,4 @@ class DateRange { } } -export default DateUtil; +export default DateRange;
{power_datum.time_to_s} {power_datum.consumption_to_s}