senaduka/old-spike
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use c3 for stacked spline

Browse Source
This commit is contained in:
Eric Hulburd
2016-03-08 15:21:16 -06:00
parent bfdebcf8f9
commit 2dd9389694
12 changed files with 249 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/controllers/houses_controller.js
View File

@@ -6,7 +6,7 @@ class HousesController {
static index(req, res){
var params = {};
if (req.query.ids) query.id = ids;
if (req.query.ids) params.id = ids;
DB.House.findAll({where: params}).then((houses)=>{
res.json({data: houses.map((house)=>{ return house.dataValues; })});
});

27
server/lib/data_helper.js Normal file
View File

@@ -0,0 +1,27 @@
import cheerio from 'cheerio';
import fs from 'fs';
import moment from 'moment';
class DataHelper {
static baseIrradiance(){
return new Promise((fnResolve, fnReject)=>{
fs.readFile(__dirname + '/../data/irradiance.html', (err, data)=>{
if (err) return fnReject(err);
var $ = cheerio.load(data),
base_irradiance = {};
$('tbody tr').each((i, elem)=>{
if (i === 0) return true;
var cells = $(elem).find('td'),
date_s = moment($(cells[1]).text(), 'YYYY-MM-DD').format('MM-DD'),
irradiance = parseFloat($(cells[2]).text());
base_irradiance[date_s] = irradiance;
});
fnResolve(base_irradiance);
});
});
}
}
export default DataHelper;

113
server/lib/tasks/design_data_generator.js Normal file
View File

@@ -0,0 +1,113 @@
import fs from 'fs';
import FsHelper from './../fs_helper';
import DB from './../../config/database';
const DESIGN_DATA_PATH = __dirname + '/../../../client/build/design/data';
class DesignDataGenerator {
constructor(house_ids, dates){
var generator = this;
generator.house_ids = house_ids;
generator.dates = dates;
}
exec(){
var generator = this;
console.log('Clearing design data directory...')
return generator.clearDirectory()
.then(()=>{
console.log('Writing house index response...')
return generator.writeHouseIndex();
})
.then(()=>{
console.log('Writing house energy and power data...');
return generator.writeHouseData();
})
.then(()=>{
console.log('Done!')
});
}
clearDirectory(){
console.log('DesignDataGenerator#clearDirectory')
return new Promise((fnResolve1, fnReject1)=>{
// remove directory & contents
FsHelper.rmdirAsync(DESIGN_DATA_PATH, ()=>{
// recreate it.
fs.mkdir(DESIGN_DATA_PATH, ()=>{
// create subdirectories
Promise.all([
new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/energy_data', fnResolve2); }),
new Promise((fnResolve2, fnReject2)=>{ fs.mkdir(DESIGN_DATA_PATH + '/power_data', fnResolve2); })
]).then(fnResolve1);
});
});
});
}
writeHouseData(){
var generator = this;
return DB.House.findAll({where: {id: generator.house_ids}})
.then((houses)=>{
var promises = [];
// for all houses, write energy and power responses.
houses.forEach((house)=>{
promises.push(new Promise((fnResolve, fnReject)=>{
DesignDataGenerator.energyIndex({house_id: house.id, dates: [generator.dates]})
.then((json)=>{
fs.writeFile(DESIGN_DATA_PATH + `/energy_data/${house.id}.json`, json, fnResolve);
});
}));
promises.push(new Promise((fnResolve, fnReject)=>{
DesignDataGenerator.powerIndex({house_id: house.id, dates: [generator.dates]})
.then((json)=>{
fs.writeFile(DESIGN_DATA_PATH + `/power_data/${house.id}.json`, json, fnResolve);
});
}));
});
return Promise.all(promises);
});
}
writeHouseIndex(){
var generator = this;
return new Promise((fnResolve, fnReject)=>{
DesignDataGenerator.housesIndex({id: generator.house_ids, dates: generator.dates})
.then((json)=>{
fs.writeFile(DESIGN_DATA_PATH + '/houses.json', json, fnResolve);
});
});
}
static housesIndex(opts){
return DB.House.findAll({where: {id: opts.id}})
.then((houses_data)=>{
if (opts.dates){
houses_data.forEach((house_datum)=>{
house_datum.data_from = opts.dates[0];
house_datum.data_until = opts.dates[1];
});
}
return JSON.stringify(houses_data, null, 2);
});
}
static powerIndex(opts){
return DB.PowerDatum.exposeForHouseAtDates(opts.house_id, opts.dates)
.then((power_data)=>{
return JSON.stringify({data: power_data});
});
}
static energyIndex(opts){
return DB.EnergyDatum.exposeForHouseAtDates(opts.house_id, opts.dates)
.then((energy_data)=>{
return JSON.stringify({data: energy_data});
});
}
}
export default DesignDataGenerator;

56
server/lib/tasks/seed_data.js
View File

@@ -5,20 +5,22 @@ import fs from 'fs';
import MathUtils from "./../../../shared/utils/math"
import DB from './../../config/database';
const DATA_PATH = __dirname + '/../../../shared/data/'
const DATA_PATH = __dirname + '/../../data/'
export class PowerDataSeed {
static saveCsv(opts, done){
opts = extend({
path: DATA_PATH + "power_data.csv"
path: "power_data.csv",
house_id: null
}, opts || {});
opts.path = DATA_PATH + opts.path;
var stream = fs.createReadStream(opts.path),
csvStream = csv.fromStream(stream, {headers: ['house_id', 'time', 'consumption', 'production']}),
csvStream = csv.fromStream(stream, {headers: ['time', 'consumption', 'production']}),
rows = [];
csvStream.on("data", function(data){
data.time = data.time;
data.house_id = opts.house_id
rows.push(data);
if (rows.length % 100 === 0){
DB.PowerDatum.bulkCreate(rows, {validate: true}).catch((error)=>{
@@ -31,13 +33,8 @@ export class PowerDataSeed {
csvStream.on("end", function(){
console.log("all rows parsed")
DB.PowerDatum.bulkCreate(rows, {validate: true}).then(()=>{
return DB.House.findAll().then((houses)=>{
var promises = [];
for (var house of houses){
var p = house.aggregatePowerToEnergyData();
promises.push(p);
}
return Promise.all(promises);
return DB.House.findOne({where: {id: opts.house_id}}).then((house)=>{
return house.aggregatePowerToEnergyData();
});
}).then(()=>{
console.log("DONE!")
@@ -51,32 +48,31 @@ export class PowerDataSeed {
end_date: moment().unix(),
interval: 900, // every 15 minutes (in s)
average: 1400, // Wh
path: DATA_PATH + "power_data.csv"
path: "power_data.csv"
}, opts || {});
console.log(opts.start_date, opts.end_date)
opts.path = DATA_PATH + opts.path;
opts.production_multiplier = parseFloat(opts.production_multiplier);
var row_date = opts.start_date,
csvStream = csv.format({headers: true}),
writableStream = fs.createWriteStream(opts.path),
house_ids = opts.house_ids.split(",")
writableStream = fs.createWriteStream(opts.path);
DB.House.findAll({where: {id: house_ids}}).then((houses)=>{
csvStream.pipe(writableStream);
writableStream.on("finish", ()=>{
console.log("DONE!")
done();
});
csvStream.pipe(writableStream);
writableStream.on("finish", ()=>{
console.log("DONE!")
done();
});
while (row_date <= opts.end_date){
for (var house of houses){
DB.House.findOne({where: {id: opts.house_id}})
.then((house)=>{
while (row_date <= opts.end_date){
var consumption = MathUtils.normal(opts.average),
production = MathUtils.normal(opts.average) * house.productionMultiplier(row_date * 1000);
csvStream.write([house.id, row_date, consumption, production]);
csvStream.write([row_date, consumption, production]);
row_date += opts.interval;
}
row_date += opts.interval;
}
csvStream.end();
});
csvStream.end();
})
}
}
@@ -86,7 +82,7 @@ export class HouseSeed {
path: DATA_PATH + "houses.csv"
}, opts || {});
var stream = fs.createReadStream(opts.path),
csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'timezone']}),
csvStream = csv.fromStream(stream, {headers: ['id', 'name', 'production_multiplier', 'timezone']}),
rows = [];
csvStream.on("data", function(data){

3
server/models/energy_datum.js
View File

@@ -17,6 +17,7 @@ var EnergyDatum = DB.sequelize.define(NAME, {
day: {
type: DB.Sequelize.INTEGER,
},
irradiance: DB.Sequelize.FLOAT,
production: DB.Sequelize.FLOAT,
consumption: DB.Sequelize.FLOAT
}, {
@@ -34,8 +35,6 @@ var EnergyDatum = DB.sequelize.define(NAME, {
exposeForHouseAtDates: (house_id, dates)=>{
var params = {house_id: house_id};
extend(params, ApiHelper.datesParamToSequelize(dates, 'day'));
console.log('EnergyDatum#exposeForHouseAtDates')
console.log(params, dates)
return EnergyDatum.findAll({
where: params,
attributes: ['id', 'production', 'consumption', 'day']

40
server/models/house.js
View File

@@ -1,5 +1,6 @@
import moment from 'moment-timezone';
import DB from "./../config/database";
import DataHelper from './../lib/data_helper';
const NAME = 'House';
@@ -15,6 +16,7 @@ var House = DB.sequelize.define(NAME, {
},
timezone: DB.Sequelize.STRING,
name: DB.Sequelize.STRING,
production_multiplier: DB.Sequelize.FLOAT,
data_until: {
type: DB.Sequelize.INTEGER,
},
@@ -33,37 +35,53 @@ var House = DB.sequelize.define(NAME, {
if (minute > 420 && minute < 1140){
multiplier = 1 - Math.abs(780 - minute) / 360;
}
return multiplier;
return multiplier * house.production_multiplier;
},
unixToLocalDay: function(unix){
var house = this;
return moment.tz(unix * 1000, house.timezone).startOf('day').unix();
},
aggregatePowerToEnergyData: function(){
dayToMonthDayString: function(unix){
var house = this;
return moment.tz(unix * 1000, house.timezone).format('MM-DD');
},
aggregatePowerToEnergyData: function(){
var house = this,
base_irradiance;
return DB.EnergyDatum.destroy({where: {house_id: house.id}})
.then(()=>{
return DataHelper.baseIrradiance();
})
.then((data)=>{
base_irradiance = data;
return DB.PowerDatum.count({where: {house_id: house.id}})
})
.then((count)=>{
var limit = 0,
energy_data = new Map(),
promises = [];
console.log('House#aggregatePowerToEnergyData')
while (limit < count){
let complete = DB.PowerDatum.findAll({where: {house_id: house.id}, limit: 1000, offset: limit, order: 'id ASC'})
.then((power_data)=>{
power_data.forEach((power_datum)=>{
var day = house.unixToLocalDay(power_datum.time),
energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
energy_data.set(day, energy_datum);
});
});
.then((power_data)=>{
power_data.forEach((power_datum)=>{
var day = house.unixToLocalDay(power_datum.time),
energy_datum = energy_data.get(day) || {production: 0, consumption: 0, day: day, house_id: house.id};
energy_datum.production += power_datum.production / 1000; // convert Wh to kWh
energy_datum.consumption += power_datum.consumption / 1000; // convert Wh to kWh
energy_data.set(day, energy_datum);
});
});
promises.push(complete);
limit += 1000;
}
return Promise.all(promises).then(()=>{
Array.from(energy_data.values()).forEach((energy_datum)=>{
let day_multiplier = 1 + Math.random() * 0.10 - 0.05,
base_day_irradiance = base_irradiance[house.dayToMonthDayString(energy_datum.day)] || 105,
irradiance = base_day_irradiance * house.production_multiplier * day_multiplier;
energy_datum.irradiance = irradiance;
})
return DB.EnergyDatum.bulkCreate(Array.from(energy_data.values()), {validate: true});
});
})