diff --git a/app/crawler/savers/postgres.js b/app/crawler/savers/postgres.js index 344e4ac..97ba285 100644 --- a/app/crawler/savers/postgres.js +++ b/app/crawler/savers/postgres.js @@ -1,6 +1,7 @@ const moment = require("moment"); const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate"); +const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory"); class PostgresSaver { connect() { @@ -11,6 +12,21 @@ class PostgresSaver { async save(results) { const savedRecords = await bulkUpsertRealEstates(results); + //Extruding data for price history table + const resultPrices = savedRecords.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + + return { + realEstateId: realEstate.dataValues.id, + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt + }; + }); + const savedPrices = await bulkUpsertPriceHistory(resultPrices); if (Array.isArray(savedRecords)) { const newRealEstates = []; diff --git a/app/helpers/db/priceHistory.js b/app/helpers/db/priceHistory.js new file mode 100644 index 0000000..3e8f69e --- /dev/null +++ b/app/helpers/db/priceHistory.js @@ -0,0 +1,20 @@ +"use strict"; +const db = require("../../models/index"); +const sequelize = require("sequelize"); + +const bulkUpsertPriceHistory = async priceHistoryData => { + try { + const order = [["realEstateId", "desc"]]; + + return await db.PriceHistory.bulkCreate(priceHistoryData, { + order, + ignoreDuplicates: true + }); + } catch (e) { + console.log("Error bulk upserting priceHistory : ", e); + } +}; + +module.exports = { + bulkUpsertPriceHistory +}; diff --git a/app/migrations/20200121000524-add-priceHistory-table.js b/app/migrations/20200121000524-add-priceHistory-table.js new file mode 100644 index 0000000..6d56f99 --- /dev/null +++ b/app/migrations/20200121000524-add-priceHistory-table.js @@ -0,0 +1,42 @@ +"use strict"; + +module.exports = { + up: (queryInterface, Sequelize) => { + const tableFields = { + id: { + type: Sequelize.BIGINT, + autoIncrement: true, + allowNull: false, + primaryKey: true + }, + realEstateId: { + type: Sequelize.BIGINT, + allowNull: false, + unique: "uniquePriceRealEstate", + references: { + model: "RealEstates", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: { + type: Sequelize.REAL, + unique: "uniquePriceRealEstate" + }, + createdAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.literal("NOW()") + }, + updatedAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.literal("NOW()") + } + }; + return queryInterface.createTable("PriceHistory", tableFields); + }, + + down: queryInterface => { + return queryInterface.dropTable("PriceHistory", {}); + } +}; diff --git a/app/migrations/20200121094500-add-constraint-priceHistory.js b/app/migrations/20200121094500-add-constraint-priceHistory.js new file mode 100644 index 0000000..45c83c3 --- /dev/null +++ b/app/migrations/20200121094500-add-constraint-priceHistory.js @@ -0,0 +1,10 @@ +"use strict"; +module.exports = { + up: (queryInterface, Sequelize) => + queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], { + type: "unique", + name: "uniquePriceRealEstate" + }), + down: queryInterface => + queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate") +}; diff --git a/app/models/priceHistory.js b/app/models/priceHistory.js new file mode 100644 index 0000000..495be53 --- /dev/null +++ b/app/models/priceHistory.js @@ -0,0 +1,44 @@ +"use strict"; + +module.exports = (sequalize, DataTypes) => { + const PriceHistory = sequalize.define( + "PriceHistory", + { + id: { + type: DataTypes.BIGINT, + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + realEstateId: { + type: DataTypes.BIGINT, + allowNull: false, + unique: "uniquePriceRealEstate", + references: { + model: "RealEstates", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: { + type: DataTypes.REAL, + unique: "uniquePriceRealEstate" + } + }, + { + freezeTableName: true + } + ); + + PriceHistory.associate = models => { + PriceHistory.hasMany(models.RealEstate, { + foreignKey: "id", + sourceKey: "realEstateId", + targetKey: "id", + as: "realEstates" + }); + }; + + return PriceHistory; +}; diff --git a/app/seeders/20200121150443-initial-price-history.js b/app/seeders/20200121150443-initial-price-history.js new file mode 100644 index 0000000..0a9531a --- /dev/null +++ b/app/seeders/20200121150443-initial-price-history.js @@ -0,0 +1,34 @@ +"use strict"; + +const { RealEstate } = require("../models"); + +module.exports = { + async up(queryInterface, Sequelize) { + //Reading initial data from RealEstate table in db + const realEstateInitialData = await RealEstate.findAll(); + //Extruding data for table PriceHistory + const priceHistoryInitialData = realEstateInitialData.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + + return { + realEstateId: realEstate.dataValues.id, + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt + }; + }); + + return queryInterface.bulkInsert( + "PriceHistory", + priceHistoryInitialData, + {} + ); + }, + + async down(queryInterface, Sequelize) { + return queryInterface.bulkDelete("PriceHistory", null, {}); + } +}; diff --git a/package.json b/package.json index 75a7cc4..cdb2c5c 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "start": "node ./index.js", "start-mon": "nodemon ./index.js", "migrate": "cd app && npx sequelize db:migrate", + "seed": "cd app && npx sequelize db:seed:all", "setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 10 && npm run migrate", "docker-start": "docker start pg_marketalerts", "docker-stop": "docker stop pg_marketalerts",