diff --git a/app/crawler/savers/postgres.js b/app/crawler/savers/postgres.js index 8c1553f..8e31f9d 100644 --- a/app/crawler/savers/postgres.js +++ b/app/crawler/savers/postgres.js @@ -14,9 +14,16 @@ class PostgresSaver { const savedRecords = await bulkUpsertRealEstates(results); //Extruding data for price history table const resultPrices = savedRecords.map(realEstate => { + //Null values canot be recognized by ignore duplicates in sequalize + //Value price = 0 indicates 'cijena na upit' + const priceTmp = + realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price; + return { realEstateId: realEstate.dataValues.id, - price: realEstate.dataValues.price + price: priceTmp, + createdAt: realEstate.dataValues.createdAt, + updatedAt: realEstate.dataValues.updatedAt }; }); const savedPrices = await bulkUpsertPriceHistory(resultPrices); diff --git a/app/helpers/db/priceHistory.js b/app/helpers/db/priceHistory.js index 9d62be8..3e8f69e 100644 --- a/app/helpers/db/priceHistory.js +++ b/app/helpers/db/priceHistory.js @@ -7,7 +7,8 @@ const bulkUpsertPriceHistory = async priceHistoryData => { const order = [["realEstateId", "desc"]]; return await db.PriceHistory.bulkCreate(priceHistoryData, { - order + order, + ignoreDuplicates: true }); } catch (e) { console.log("Error bulk upserting priceHistory : ", e); diff --git a/app/migrations/20200121094500-add-constraint-priceHistory.js b/app/migrations/20200121094500-add-constraint-priceHistory.js new file mode 100644 index 0000000..45c83c3 --- /dev/null +++ b/app/migrations/20200121094500-add-constraint-priceHistory.js @@ -0,0 +1,10 @@ +"use strict"; +module.exports = { + up: (queryInterface, Sequelize) => + queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], { + type: "unique", + name: "uniquePriceRealEstate" + }), + down: queryInterface => + queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate") +}; diff --git a/app/models/priceHistory.js b/app/models/priceHistory.js index 560a4fa..495be53 100644 --- a/app/models/priceHistory.js +++ b/app/models/priceHistory.js @@ -1,29 +1,35 @@ "use strict"; module.exports = (sequalize, DataTypes) => { - const PriceHistory = sequalize.define("PriceHistory", { - id: { - type: DataTypes.BIGINT, - autoIncrement: true, - primaryKey: true, - allowNull: false - }, - realEstateId: { - type: DataTypes.BIGINT, - allowNull: false, - unique: "uniquePriceRealEstate", - references: { - model: "RealEstates", - key: "id" + const PriceHistory = sequalize.define( + "PriceHistory", + { + id: { + type: DataTypes.BIGINT, + autoIncrement: true, + primaryKey: true, + allowNull: false }, - onUpdate: "CASCADE", - onDelete: "SET NULL" + realEstateId: { + type: DataTypes.BIGINT, + allowNull: false, + unique: "uniquePriceRealEstate", + references: { + model: "RealEstates", + key: "id" + }, + onUpdate: "CASCADE", + onDelete: "SET NULL" + }, + price: { + type: DataTypes.REAL, + unique: "uniquePriceRealEstate" + } }, - price: { - type: DataTypes.REAL, - unique: "uniquePriceRealEstate" + { + freezeTableName: true } - }); + ); PriceHistory.associate = models => { PriceHistory.hasMany(models.RealEstate, { diff --git a/app/seeders/20200121150443-initial-price-history.js b/app/seeders/20200121150443-initial-price-history.js new file mode 100644 index 0000000..9d2f5b0 --- /dev/null +++ b/app/seeders/20200121150443-initial-price-history.js @@ -0,0 +1,19 @@ +"use strict"; + +module.exports = { + up: (queryInterface, Sequelize) => { + // ?? + const realEstateInitialData = await queryInterface.Sequelize.findall (); + const priceHistoryInitialData = realEstateInitialData.map (); + + return queryInterface.bulkInsert( + "PriceHistory", + priceHistoryInitialData, + {} + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.bulkDelete("PriceHistory", null, {}); + } +};