Added price history log.

app/common/enums.js

@@ -7,42 +7,7 @@ const PRICE_SLIDER_OPTIONS_SALE = {
   step: 1000,
   connect: true
 };
-const FLAT_PRICE_SLIDER_OPTIONS_SALE = {
-  start: [50000, 150000],
-  range: {
-    min: [0],
-    max: [800000]
-  },
-  step: 5000,
-  connect: true
-};
-const HOUSE_PRICE_SLIDER_OPTIONS_SALE = {
-  start: [50000, 150000],
-  range: {
-    min: [0],
-    max: [1500000]
-  },
-  step: 10000,
-  connect: true
-};
-const OFFICE_PRICE_SLIDER_OPTIONS_SALE = {
-  start: [15000, 50000],
-  range: {
-    min: [0],
-    max: [2000000]
-  },
-  step: 2000,
-  connect: true
-};
-const LAND_PRICE_SLIDER_OPTIONS_SALE = {
-  start: [40000, 80000],
-  range: {
-    min: [0],
-    max: [2000000]
-  },
-  step: 10000,
-  connect: true
-};
+
 const PRICE_SLIDER_OPTIONS_RENT = {
   start: [300, 500],
   range: {
@@ -52,62 +17,18 @@ const PRICE_SLIDER_OPTIONS_RENT = {
   step: 50,
   connect: true
 };
-const FLAT_PRICE_SLIDER_OPTIONS_RENT = {
-  start: [300, 600],
-  range: {
-    min: [0],
-    max: [4000]
-  },
-  step: 100,
-  connect: true
-};
-const HOUSE_PRICE_SLIDER_OPTIONS_RENT = {
-  start: [500, 1000],
-  range: {
-    min: [0],
-    max: [10000]
-  },
-  step: 100,
-  connect: true
-};
-const OFFICE_PRICE_SLIDER_OPTIONS_RENT = {
-  start: [200, 1000],
-  range: {
-    min: [0],
-    max: [20000]
-  },
-  step: 100,
-  connect: true
-};
-const LAND_PRICE_SLIDER_OPTIONS_RENT = {
-  start: [500, 1000],
-  range: {
-    min: [0],
-    max: [20000]
-  },
-  step: 100,
-  connect: true
-};
+
 //This will be used for Flats, Apartments, Houses
 const HOME_SIZE_SLIDER_OPTIONS = {
   start: [30, 75],
   range: {
     min: [0],
-    max: [500]
+    max: [400]
   },
   step: 5,
   connect: true
 };
 
-const OFFICE_SIZE_SLIDER_OPTIONS = {
-  start: [30, 150],
-  range: {
-    min: [0],
-    max: [1200]
-  },
-  step: 10,
-  connect: true
-};
 const GARDEN_SIZE_SLIDER_OPTIONS = {
   start: [100, 1000],
   range: {
@@ -190,8 +111,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   HOUSE: {
@@ -205,8 +126,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: true,
     hasFloorProp: false,
-    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   },
@@ -221,9 +142,9 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: OFFICE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: OFFICE_PRICE_SLIDER_OPTIONS_RENT,
-    sizeSliderOptions: OFFICE_SIZE_SLIDER_OPTIONS
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   LAND: {
     id: "LAND",
@@ -236,8 +157,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: false,
     hasNumberOfFloors: false,
     hasFloorProp: false,
-    priceSliderOptionsSale: LAND_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: LAND_PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
   },
   APARTMENT: {
@@ -251,8 +172,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   GARAGE: {
@@ -281,8 +202,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: true,
     hasFloorProp: false,
-    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   }

app/crawler/savers/postgres.js

@@ -1,6 +1,7 @@
 const moment = require("moment");
 
 const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
+const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory");
 
 class PostgresSaver {
   connect() {
@@ -11,6 +12,21 @@ class PostgresSaver {
 
   async save(results) {
     const savedRecords = await bulkUpsertRealEstates(results);
+    //Extruding data for price history table
+    const resultPrices = savedRecords.map(realEstate => {
+      //Null values canot be recognized by ignore duplicates in sequalize
+      //Value price = 0 indicates 'cijena na upit'
+      const priceTmp =
+        realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price;
+
+      return {
+        realEstateId: realEstate.dataValues.id,
+        price: priceTmp,
+        createdAt: realEstate.dataValues.createdAt,
+        updatedAt: realEstate.dataValues.updatedAt
+      };
+    });
+    const savedPrices = await bulkUpsertPriceHistory(resultPrices);
 
     if (Array.isArray(savedRecords)) {
       const newRealEstates = [];

app/helpers/db/priceHistory.js

@@ -0,0 +1,20 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+
+const bulkUpsertPriceHistory = async priceHistoryData => {
+  try {
+    const order = [["realEstateId", "desc"]];
+
+    return await db.PriceHistory.bulkCreate(priceHistoryData, {
+      order,
+      ignoreDuplicates: true
+    });
+  } catch (e) {
+    console.log("Error bulk upserting priceHistory : ", e);
+  }
+};
+
+module.exports = {
+  bulkUpsertPriceHistory
+};

app/migrations/20200121000524-add-priceHistory-table.js

@@ -0,0 +1,42 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false,
+        primaryKey: true
+      },
+      realEstateId: {
+        type: Sequelize.BIGINT,
+        allowNull: false,
+        unique: "uniquePriceRealEstate",
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      price: {
+        type: Sequelize.REAL,
+        unique: "uniquePriceRealEstate"
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("PriceHistory", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("PriceHistory", {});
+  }
+};

app/migrations/20200121094500-add-constraint-priceHistory.js

@@ -0,0 +1,10 @@
+"use strict";
+module.exports = {
+  up: (queryInterface, Sequelize) =>
+    queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], {
+      type: "unique",
+      name: "uniquePriceRealEstate"
+    }),
+  down: queryInterface =>
+    queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate")
+};

app/models/priceHistory.js

@@ -0,0 +1,44 @@
+"use strict";
+
+module.exports = (sequalize, DataTypes) => {
+  const PriceHistory = sequalize.define(
+    "PriceHistory",
+    {
+      id: {
+        type: DataTypes.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false
+      },
+      realEstateId: {
+        type: DataTypes.BIGINT,
+        allowNull: false,
+        unique: "uniquePriceRealEstate",
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      price: {
+        type: DataTypes.REAL,
+        unique: "uniquePriceRealEstate"
+      }
+    },
+    {
+      freezeTableName: true
+    }
+  );
+
+  PriceHistory.associate = models => {
+    PriceHistory.hasMany(models.RealEstate, {
+      foreignKey: "id",
+      sourceKey: "realEstateId",
+      targetKey: "id",
+      as: "realEstates"
+    });
+  };
+
+  return PriceHistory;
+};

app/seeders/20200121150443-initial-price-history.js

@@ -0,0 +1,34 @@
+"use strict";
+
+const { RealEstate } = require("../models");
+
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    //Reading initial data from RealEstate table in db
+    const realEstateInitialData = await RealEstate.findAll();
+    //Extruding data for table PriceHistory
+    const priceHistoryInitialData = realEstateInitialData.map(realEstate => {
+      //Null values canot be recognized by ignore duplicates in sequalize
+      //Value price = 0 indicates 'cijena na upit'
+      const priceTmp =
+        realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price;
+
+      return {
+        realEstateId: realEstate.dataValues.id,
+        price: priceTmp,
+        createdAt: realEstate.dataValues.createdAt,
+        updatedAt: realEstate.dataValues.updatedAt
+      };
+    });
+
+    return queryInterface.bulkInsert(
+      "PriceHistory",
+      priceHistoryInitialData,
+      {}
+    );
+  },
+
+  async down(queryInterface, Sequelize) {
+    return queryInterface.bulkDelete("PriceHistory", null, {});
+  }
+};

package.json

@@ -8,6 +8,7 @@
     "start": "node ./index.js",
     "start-mon": "nodemon ./index.js",
     "migrate": "cd app && npx sequelize db:migrate",
+    "seed": "cd app && npx sequelize db:seed:all",
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 10 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",