From 0a181f742f9e4478ffefe4a778108ea5cb95d0cd Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Mon, 20 Jan 2020 23:09:02 +0100 Subject: [PATCH 1/4] WIP Added model and migration for new table priceHistory. --- .../20200120215830-add-priceHistory-table.js | 38 +++++++++++++++++++ app/models/priceHistory.js | 35 +++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 app/migrations/20200120215830-add-priceHistory-table.js create mode 100644 app/models/priceHistory.js diff --git a/app/migrations/20200120215830-add-priceHistory-table.js b/app/migrations/20200120215830-add-priceHistory-table.js new file mode 100644 index 0000000..eb8f616 --- /dev/null +++ b/app/migrations/20200120215830-add-priceHistory-table.js @@ -0,0 +1,38 @@ +"use strict"; + +module.exports = { + up: (queryInterface, Sequelize) => { + const tableFields = { + id: { + type: Sequelize.BIGINT, + autoIncrement: true, + allowNull: false, + primaryKey: true + }, + realEstateId: { + type: Sequelize.BIGINT, + allowNull: false, + references: { + model: "RealEstate", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: Sequelize.REAL, + createdAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.literal("NOW()") + }, + updatedAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.literal("NOW()") + } + }; + return queryInterface.createTable("PriceHistory", tableFields); + }, + + down: queryInterface => { + return queryInterface.dropTable("PriceHistory", {}); + } +}; diff --git a/app/models/priceHistory.js b/app/models/priceHistory.js new file mode 100644 index 0000000..ea9ddea --- /dev/null +++ b/app/models/priceHistory.js @@ -0,0 +1,35 @@ +"use strict"; + +module.exports = (sequalize, DataTypes) => { + const PriceHistory = sequalize.define("PriceHistory", { + id: { + type: DataTypes.BIGINT, + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + realEstateId: { + type: DataTypes.BIGINT, + allowNull: false, + + references: { + model: "RealEstate", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: DataTypes.REAL + }); + + PriceHistory.associate = models => { + PriceHistory.hasMany(models.RealEstate, { + foreignKey: "id", + sourceKey: "realEstateId", + targetKey: "id", + as: "realEstates" + }); + }; + + return PriceHistory; +}; -- 2.47.3 From 42eddb3aa5e6899da304be787b21dae9b38881f0 Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Tue, 21 Jan 2020 01:19:35 +0100 Subject: [PATCH 2/4] WIP Bulk create not working. --- app/crawler/savers/postgres.js | 13 ++++++++++++- app/helpers/db/priceHistory.js | 19 +++++++++++++++++++ app/helpers/db/realEstate.js | 4 +++- ... 20200121000524-add-priceHistory-table.js} | 8 ++++++-- app/models/priceHistory.js | 9 ++++++--- 5 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 app/helpers/db/priceHistory.js rename app/migrations/{20200120215830-add-priceHistory-table.js => 20200121000524-add-priceHistory-table.js} (83%) diff --git a/app/crawler/savers/postgres.js b/app/crawler/savers/postgres.js index 344e4ac..8c1553f 100644 --- a/app/crawler/savers/postgres.js +++ b/app/crawler/savers/postgres.js @@ -1,6 +1,7 @@ const moment = require("moment"); const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate"); +const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory"); class PostgresSaver { connect() { @@ -11,7 +12,17 @@ class PostgresSaver { async save(results) { const savedRecords = await bulkUpsertRealEstates(results); - + //Extruding data for price history table + const resultPrices = savedRecords.map(realEstate => { + return { + realEstateId: realEstate.dataValues.id, + price: realEstate.dataValues.price + }; + }); + const savedPrices = await bulkUpsertPriceHistory(resultPrices); + // + console.log("savedPrices", savedPrices); + // if (Array.isArray(savedRecords)) { const newRealEstates = []; const existingRealEstates = []; diff --git a/app/helpers/db/priceHistory.js b/app/helpers/db/priceHistory.js new file mode 100644 index 0000000..9d62be8 --- /dev/null +++ b/app/helpers/db/priceHistory.js @@ -0,0 +1,19 @@ +"use strict"; +const db = require("../../models/index"); +const sequelize = require("sequelize"); + +const bulkUpsertPriceHistory = async priceHistoryData => { + try { + const order = [["realEstateId", "desc"]]; + + return await db.PriceHistory.bulkCreate(priceHistoryData, { + order + }); + } catch (e) { + console.log("Error bulk upserting priceHistory : ", e); + } +}; + +module.exports = { + bulkUpsertPriceHistory +}; diff --git a/app/helpers/db/realEstate.js b/app/helpers/db/realEstate.js index ebeb84c..cb7a2ef 100644 --- a/app/helpers/db/realEstate.js +++ b/app/helpers/db/realEstate.js @@ -63,7 +63,9 @@ const bulkUpsertRealEstates = async realEstateData => { "numberOfViewsAgency" ]; const order = [["updatedAt", "desc"]]; - + // + //console.log("realEstateData:", realEstateData); + // return await db.RealEstate.bulkCreate(realEstateData, { updateOnDuplicate: fieldsToUpdateIfDuplicate, returning: true, diff --git a/app/migrations/20200120215830-add-priceHistory-table.js b/app/migrations/20200121000524-add-priceHistory-table.js similarity index 83% rename from app/migrations/20200120215830-add-priceHistory-table.js rename to app/migrations/20200121000524-add-priceHistory-table.js index eb8f616..6d56f99 100644 --- a/app/migrations/20200120215830-add-priceHistory-table.js +++ b/app/migrations/20200121000524-add-priceHistory-table.js @@ -12,14 +12,18 @@ module.exports = { realEstateId: { type: Sequelize.BIGINT, allowNull: false, + unique: "uniquePriceRealEstate", references: { - model: "RealEstate", + model: "RealEstates", key: "id" }, onUpdate: "CASCADE", onDelete: "SET NULL" }, - price: Sequelize.REAL, + price: { + type: Sequelize.REAL, + unique: "uniquePriceRealEstate" + }, createdAt: { type: Sequelize.DATE, defaultValue: Sequelize.literal("NOW()") diff --git a/app/models/priceHistory.js b/app/models/priceHistory.js index ea9ddea..560a4fa 100644 --- a/app/models/priceHistory.js +++ b/app/models/priceHistory.js @@ -11,15 +11,18 @@ module.exports = (sequalize, DataTypes) => { realEstateId: { type: DataTypes.BIGINT, allowNull: false, - + unique: "uniquePriceRealEstate", references: { - model: "RealEstate", + model: "RealEstates", key: "id" }, onUpdate: "CASCADE", onDelete: "SET NULL" }, - price: DataTypes.REAL + price: { + type: DataTypes.REAL, + unique: "uniquePriceRealEstate" + } }); PriceHistory.associate = models => { -- 2.47.3 From 8d3f001678d29d8a69aa5baf91551e8c8f8ec32d Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Tue, 21 Jan 2020 16:28:47 +0100 Subject: [PATCH 3/4] WIP Added model, migration and bulk upsert fnc. --- app/crawler/savers/postgres.js | 9 +++- app/helpers/db/priceHistory.js | 3 +- ...00121094500-add-constraint-priceHistory.js | 10 ++++ app/models/priceHistory.js | 46 +++++++++++-------- .../20200121150443-initial-price-history.js | 19 ++++++++ 5 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 app/migrations/20200121094500-add-constraint-priceHistory.js create mode 100644 app/seeders/20200121150443-initial-price-history.js diff --git a/app/crawler/savers/postgres.js b/app/crawler/savers/postgres.js index 8c1553f..8e31f9d 100644 --- a/app/crawler/savers/postgres.js +++ b/app/crawler/savers/postgres.js @@ -14,9 +14,16 @@ class PostgresSaver { const savedRecords = await bulkUpsertRealEstates(results); //Extruding data for price history table const resultPrices = savedRecords.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + return { realEstateId: realEstate.dataValues.id, - price: realEstate.dataValues.price + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt }; }); const savedPrices = await bulkUpsertPriceHistory(resultPrices); diff --git a/app/helpers/db/priceHistory.js b/app/helpers/db/priceHistory.js index 9d62be8..3e8f69e 100644 --- a/app/helpers/db/priceHistory.js +++ b/app/helpers/db/priceHistory.js @@ -7,7 +7,8 @@ const bulkUpsertPriceHistory = async priceHistoryData => { const order = [["realEstateId", "desc"]]; return await db.PriceHistory.bulkCreate(priceHistoryData, { - order + order, + ignoreDuplicates: true }); } catch (e) { console.log("Error bulk upserting priceHistory : ", e); diff --git a/app/migrations/20200121094500-add-constraint-priceHistory.js b/app/migrations/20200121094500-add-constraint-priceHistory.js new file mode 100644 index 0000000..45c83c3 --- /dev/null +++ b/app/migrations/20200121094500-add-constraint-priceHistory.js @@ -0,0 +1,10 @@ +"use strict"; +module.exports = { + up: (queryInterface, Sequelize) => + queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], { + type: "unique", + name: "uniquePriceRealEstate" + }), + down: queryInterface => + queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate") +}; diff --git a/app/models/priceHistory.js b/app/models/priceHistory.js index 560a4fa..495be53 100644 --- a/app/models/priceHistory.js +++ b/app/models/priceHistory.js @@ -1,29 +1,35 @@ "use strict"; module.exports = (sequalize, DataTypes) => { - const PriceHistory = sequalize.define("PriceHistory", { - id: { - type: DataTypes.BIGINT, - autoIncrement: true, - primaryKey: true, - allowNull: false - }, - realEstateId: { - type: DataTypes.BIGINT, - allowNull: false, - unique: "uniquePriceRealEstate", - references: { - model: "RealEstates", - key: "id" + const PriceHistory = sequalize.define( + "PriceHistory", + { + id: { + type: DataTypes.BIGINT, + autoIncrement: true, + primaryKey: true, + allowNull: false }, - onUpdate: "CASCADE", - onDelete: "SET NULL" + realEstateId: { + type: DataTypes.BIGINT, + allowNull: false, + unique: "uniquePriceRealEstate", + references: { + model: "RealEstates", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: { + type: DataTypes.REAL, + unique: "uniquePriceRealEstate" + } }, - price: { - type: DataTypes.REAL, - unique: "uniquePriceRealEstate" + { + freezeTableName: true } - }); + ); PriceHistory.associate = models => { PriceHistory.hasMany(models.RealEstate, { diff --git a/app/seeders/20200121150443-initial-price-history.js b/app/seeders/20200121150443-initial-price-history.js new file mode 100644 index 0000000..9d2f5b0 --- /dev/null +++ b/app/seeders/20200121150443-initial-price-history.js @@ -0,0 +1,19 @@ +"use strict"; + +module.exports = { + up: (queryInterface, Sequelize) => { + // ?? + const realEstateInitialData = await queryInterface.Sequelize.findall (); + const priceHistoryInitialData = realEstateInitialData.map (); + + return queryInterface.bulkInsert( + "PriceHistory", + priceHistoryInitialData, + {} + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.bulkDelete("PriceHistory", null, {}); + } +}; -- 2.47.3 From c6f0e039a58105c9f8538df18fdd0f4bcd443102 Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Tue, 21 Jan 2020 23:12:04 +0100 Subject: [PATCH 4/4] Added price history log. --- app/crawler/savers/postgres.js | 4 +-- app/helpers/db/realEstate.js | 4 +-- .../20200121150443-initial-price-history.js | 27 ++++++++++++++----- package.json | 1 + 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/app/crawler/savers/postgres.js b/app/crawler/savers/postgres.js index 8e31f9d..97ba285 100644 --- a/app/crawler/savers/postgres.js +++ b/app/crawler/savers/postgres.js @@ -27,9 +27,7 @@ class PostgresSaver { }; }); const savedPrices = await bulkUpsertPriceHistory(resultPrices); - // - console.log("savedPrices", savedPrices); - // + if (Array.isArray(savedRecords)) { const newRealEstates = []; const existingRealEstates = []; diff --git a/app/helpers/db/realEstate.js b/app/helpers/db/realEstate.js index cb7a2ef..ebeb84c 100644 --- a/app/helpers/db/realEstate.js +++ b/app/helpers/db/realEstate.js @@ -63,9 +63,7 @@ const bulkUpsertRealEstates = async realEstateData => { "numberOfViewsAgency" ]; const order = [["updatedAt", "desc"]]; - // - //console.log("realEstateData:", realEstateData); - // + return await db.RealEstate.bulkCreate(realEstateData, { updateOnDuplicate: fieldsToUpdateIfDuplicate, returning: true, diff --git a/app/seeders/20200121150443-initial-price-history.js b/app/seeders/20200121150443-initial-price-history.js index 9d2f5b0..0a9531a 100644 --- a/app/seeders/20200121150443-initial-price-history.js +++ b/app/seeders/20200121150443-initial-price-history.js @@ -1,11 +1,26 @@ "use strict"; +const { RealEstate } = require("../models"); + module.exports = { - up: (queryInterface, Sequelize) => { - // ?? - const realEstateInitialData = await queryInterface.Sequelize.findall (); - const priceHistoryInitialData = realEstateInitialData.map (); - + async up(queryInterface, Sequelize) { + //Reading initial data from RealEstate table in db + const realEstateInitialData = await RealEstate.findAll(); + //Extruding data for table PriceHistory + const priceHistoryInitialData = realEstateInitialData.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + + return { + realEstateId: realEstate.dataValues.id, + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt + }; + }); + return queryInterface.bulkInsert( "PriceHistory", priceHistoryInitialData, @@ -13,7 +28,7 @@ module.exports = { ); }, - down: (queryInterface, Sequelize) => { + async down(queryInterface, Sequelize) { return queryInterface.bulkDelete("PriceHistory", null, {}); } }; diff --git a/package.json b/package.json index 75a7cc4..cdb2c5c 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "start": "node ./index.js", "start-mon": "nodemon ./index.js", "migrate": "cd app && npx sequelize db:migrate", + "seed": "cd app && npx sequelize db:seed:all", "setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 10 && npm run migrate", "docker-start": "docker start pg_marketalerts", "docker-stop": "docker stop pg_marketalerts", -- 2.47.3