Added price history log.

app/crawler/savers/postgres.js

@@ -1,6 +1,7 @@
 const moment = require("moment");
 
 const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
+const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory");
 
 class PostgresSaver {
   connect() {
@@ -11,6 +12,21 @@ class PostgresSaver {
 
   async save(results) {
     const savedRecords = await bulkUpsertRealEstates(results);
+    //Extruding data for price history table
+    const resultPrices = savedRecords.map(realEstate => {
+      //Null values canot be recognized by ignore duplicates in sequalize
+      //Value price = 0 indicates 'cijena na upit'
+      const priceTmp =
+        realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price;
+
+      return {
+        realEstateId: realEstate.dataValues.id,
+        price: priceTmp,
+        createdAt: realEstate.dataValues.createdAt,
+        updatedAt: realEstate.dataValues.updatedAt
+      };
+    });
+    const savedPrices = await bulkUpsertPriceHistory(resultPrices);
 
     if (Array.isArray(savedRecords)) {
       const newRealEstates = [];

app/helpers/db/priceHistory.js

@@ -0,0 +1,20 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+
+const bulkUpsertPriceHistory = async priceHistoryData => {
+  try {
+    const order = [["realEstateId", "desc"]];
+
+    return await db.PriceHistory.bulkCreate(priceHistoryData, {
+      order,
+      ignoreDuplicates: true
+    });
+  } catch (e) {
+    console.log("Error bulk upserting priceHistory : ", e);
+  }
+};
+
+module.exports = {
+  bulkUpsertPriceHistory
+};

app/helpers/db/realEstate.js

@@ -2,8 +2,6 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { AD_CATEGORY } = require("../../common/enums");
-
 const bulkUpsertRealEstates = async realEstateData => {
   try {
     const fieldsToUpdateIfDuplicate = [
@@ -104,9 +102,6 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     accessRoadType
   } = searchRequest;
 
-  //Needed for defining which attribute should exist or not
-  const realEstateTypeObject = AD_CATEGORY[realEstateType];
-
   const longitudeColumn = sequelize.col("locationLong");
   const latitudeColumn = sequelize.col("locationLat");
 
@@ -180,13 +175,8 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     [Op.and]: geoSearchQueryPart
   };
 
-  //Every other attribute is checked separately and included in query only if it is defined for real estate type
-
-  if (
-    realEstateTypeObject.hasGardenSize &&
-    gardenSizeMax != null &&
-    gardenSizeMin != null
-  ) {
+  //Every other attribute is checked separately and included in query only if it is defined
+  if (gardenSizeMax && gardenSizeMin) {
     query.gardenSize = {
       [Op.lte]: gardenSizeMax,
       [Op.gte]: gardenSizeMin
@@ -202,11 +192,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (
-    realEstateTypeObject.hasNumberOfRoom &&
-    numberOfRoomsMin != null &&
-    numberOfRoomsMax != null
-  ) {
+  if (numberOfRoomsMin && numberOfRoomsMax) {
     query.numberOfRooms = {
       [Op.lte]: numberOfRoomsMax,
       [Op.gte]: numberOfRoomsMin
@@ -222,11 +208,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (
-    realEstateTypeObject.hasNumberOfFloors &&
-    numberOfFloorsMin != null &&
-    numberOfFloorsMax != null
-  ) {
+  if (numberOfFloorsMin && numberOfFloorsMax) {
     query.numberOfFloors = {
       [Op.lte]: numberOfFloorsMax,
       [Op.gte]: numberOfFloorsMin
@@ -242,11 +224,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (
-    realEstateTypeObject.hasFloorProp &&
-    floorMin != null &&
-    floorMax != null
-  ) {
+  if (floorMin && floorMax) {
     query.floor = {
       [Op.lte]: floorMax,
       [Op.gte]: floorMin
@@ -261,10 +239,8 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
       }
     };
   }
-  //Logic for balcony, newBuilding and elevator from users side
-  //If true is checked, then I want characteristic to be true but,
-  //if it is not checked, then I dont care - it can be null or false or true
-  if (realEstateTypeObject.hasBalconyProp && balcony === true) {
+
+  if (balcony) {
     query.balcony = {
       [Op.eq]: balcony
     };
@@ -276,7 +252,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (realEstateTypeObject.hasNewBuildingProp && newBuilding === true) {
+  if (newBuilding) {
     query.newBuilding = {
       [Op.eq]: newBuilding
     };
@@ -288,7 +264,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (realEstateTypeObject.hasElevatorProp && elevator === true) {
+  if (elevator) {
     query.elevator = {
       [Op.eq]: elevator
     };
@@ -299,8 +275,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
       }
     };
   }
-  //If user wants 'ANY' road type acces then it is not included in query -
-  //returns every road type and null values
+
   if (accessRoadType !== "ANY") {
     query.accessRoadType = {
       [Op.eq]: accessRoadType

app/helpers/db/searchRequest.js

@@ -49,383 +49,128 @@ const findSearchRequestsForRealEstate = async realEstate => {
 
   const geoSearchQueryPart = sequelize.where(contains, true);
 
-  //Needed for defining which attribute should exist or not
-  const realEstateTypeObject = AD_CATEGORY[realEstateType];
-
-  // ?? Needed to decide on including incomplete RealEstates data
-  let checkForIncompleteWanted = false;
-
-  //Attributes are checked separately to make different query parts
-
-  //If price is null it will be excluded from query - it will show properties with null price values
-  //User always defines price and area (sliders) - not null in search req
-  let priceQuery = {};
-  if (price != null) {
-    priceQuery = {
-      [Op.and]: [
-        {
-          priceMin: {
-            [Op.lte]: price
-          }
-        },
-        {
-          priceMax: {
-            [Op.gte]: price
-          }
-        }
-      ]
-    };
-  }
-
-  let areaQuery = {};
-  if (area != null) {
-    areaQuery = {
-      [Op.and]: [
-        {
-          sizeMin: {
-            [Op.lte]: area
-          }
-        },
-        {
-          sizeMax: {
-            [Op.gte]: area
-          }
-        }
-      ]
-    };
-  } else {
-    checkForIncompleteWanted = true;
-  }
-
-  //Other attributes can be defined or not depending on RealEstate type
-  //we check what to include in query based on real estate type object
-  let gardenSizeQuery = {};
-  if (realEstateTypeObject.hasGardenSize) {
-    if (gardenSize != null) {
-      gardenSizeQuery = {
-        [Op.and]: [
-          {
-            gardenSizeMin: {
-              [Op.lte]: gardenSize
-            }
-          },
-          {
-            gardenSizeMax: {
-              [Op.gte]: gardenSize
-            }
-          }
-        ]
-      };
-    } else {
-      checkForIncompleteWanted = true;
-    }
-  }
-
-  let numberOfRoomsQuery = {};
-  if (realEstateTypeObject.hasNumberOfRoom) {
-    if (numberOfRooms != null) {
-      //If real estate has defined number of rooms ex. 3 it returns req
-      // that accepts 3 rooms or ones that don't have defined number - null
-      //Ex. they didnt choose advanced filters at all
-      numberOfRoomsQuery = {
-        [Op.and]: [
-          {
-            numberOfRoomsMin: {
-              [Op.or]: {
-                [Op.lte]: numberOfRooms,
-                [Op.is]: null
-              }
-            }
-          },
-          {
-            numberOfRoomsMax: {
-              [Op.or]: {
-                [Op.gte]: numberOfRooms,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
-      };
-    } else {
-      // If real estate dont have defined number of rooms ex. null
-      //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads
-      numberOfRoomsQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                numberOfRoomsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfRoomsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.eq]: true
-            }
-          }
-        ]
-      };
-    }
-  }
-  //Same logic for number of Floors and floors
-  let numberOfFloorsQuery = {};
-  if (realEstateTypeObject.hasNumberOfFloors) {
-    if (numberOfFloors != null) {
-      numberOfFloorsQuery = {
-        [Op.and]: [
-          {
-            numberOfFloorsMin: {
-              [Op.or]: {
-                [Op.lte]: numberOfFloors,
-                [Op.is]: null
-              }
-            }
-          },
-          {
-            numberOfFloorsMax: {
-              [Op.or]: {
-                [Op.gte]: numberOfFloors,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
-      };
-    } else {
-      numberOfFloorsQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                numberOfFloorsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfFloorsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.eq]: true
-            }
-          }
-        ]
-      };
-    }
-  }
-  let floorQuery = {};
-  if (realEstateTypeObject.hasFloorProp) {
-    if (floor != null) {
-      floorQuery = {
-        [Op.and]: [
-          {
-            floorMin: {
-              [Op.or]: {
-                [Op.lte]: floor,
-                [Op.is]: null
-              }
-            }
-          },
-          {
-            floorMax: {
-              [Op.or]: {
-                [Op.gte]: floor,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
-      };
-    } else {
-      floorQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                floorMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                floorMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.eq]: true
-            }
-          }
-        ]
-      };
-    }
-  }
-
-  //Logic for balcony, newBuilding and elevator
-  //If user dont check checkbox for ex. elevator  it does not mean he only wants no elevator
-  //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query
-  //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true
-  //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads
-  let balconyQuery = {};
-  if (realEstateTypeObject.hasBalconyProp && balcony !== true) {
-    if (balcony === false) {
-      balconyQuery = {
-        balcony: {
-          [Op.ne]: true
-        }
-      };
-    } else if (balcony === null) {
-      balconyQuery = {
-        [Op.or]: [
-          {
-            balcony: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                balcony: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.eq]: true
-                }
-              }
-            ]
-          }
-        ]
-      };
-    }
-  }
-  let newBuildingQuery = {};
-  if (realEstateTypeObject.hasNewBuildingProp && newBuilding !== true) {
-    if (newBuilding === false) {
-      newBuildingQuery = {
-        newBuilding: {
-          [Op.ne]: true
-        }
-      };
-    } else if (newBuilding === null) {
-      newBuildingQuery = {
-        [Op.or]: [
-          {
-            newBuilding: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                newBuilding: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.eq]: true
-                }
-              }
-            ]
-          }
-        ]
-      };
-    }
-  }
-  let elevatorQuery = {};
-  if (realEstateTypeObject.hasElevatorProp && elevator !== true) {
-    if (elevator === false) {
-      elevatorQuery = {
-        elevator: {
-          [Op.ne]: true
-        }
-      };
-    } else if (elevator === null) {
-      elevatorQuery = {
-        [Op.or]: [
-          {
-            elevator: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                elevator: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.eq]: true
-                }
-              }
-            ]
-          }
-        ]
-      };
-    }
-  }
-  //General query consists of each individual query
+  //General query contains only attributes that are defined for every RealEstate - not null
   const query = {
     adType,
     realEstateType,
     subscribed: true,
-    [Op.and]: [
-      geoSearchQueryPart,
-      priceQuery,
-      areaQuery,
-      gardenSizeQuery,
-      numberOfRoomsQuery,
-      numberOfFloorsQuery,
-      floorQuery,
-      balconyQuery,
-      newBuildingQuery,
-      elevatorQuery
-    ]
+    [Op.and]: geoSearchQueryPart
   };
+  //Needed for defining which attribute should exist or not
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
+  //Needed to decide on including incomplete RealEstates data
+  let checkForIncompleteWanted = false;
 
-  //AccessRoadType is defined - should exists for each ad and estate type
-  if (accessRoadType != null) {
+  //Attributes are checked separately and included in query only if defined
+  //Price and area should be defined for every property
+
+  if (price) {
+    query.priceMin = {
+      [Op.lte]: price
+    };
+    query.priceMax = {
+      [Op.gte]: price
+    };
+  }
+
+  if (area) {
+    query.sizeMin = {
+      [Op.lte]: area
+    };
+    query.sizeMax = {
+      [Op.gte]: area
+    };
+  } else {
+    checkForIncompleteWanted = true;
+  }
+  //Other attributes can be defined or not depending on RealEstate type
+  if (gardenSize) {
+    query.gardenSizeMin = {
+      [Op.lte]: gardenSize
+    };
+    query.gardenSizeMax = {
+      [Op.gte]: gardenSize
+    };
+  } else if (realEstateTypeObject.hasGardenSize) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (numberOfRooms) {
+    query.numberOfRoomsMin = {
+      [Op.lte]: numberOfRooms
+    };
+    query.numberOfRoomsMax = {
+      [Op.gte]: numberOfRooms
+    };
+  } else if (realEstateTypeObject.hasNumberOfRoom) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (numberOfFloors) {
+    query.numberOfFloorsMin = {
+      [Op.lte]: numberOfFloors
+    };
+    query.numberOfFloorsMax = {
+      [Op.gte]: numberOfFloors
+    };
+  } else if (realEstateTypeObject.hasNumberOfFloors) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (floor) {
+    query.floorMin = {
+      [Op.lte]: floor
+    };
+    query.floorMax = {
+      [Op.gte]: floor
+    };
+  } else if (realEstateTypeObject.hasFloorProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (accessRoadType) {
     query.accessRoadType = {
       [Op.or]: {
-        [Op.like]: "ANY",
+        [Op.eq]: "ANY",
         [Op.eq]: accessRoadType
       }
     };
-  } else {
-    //Null values are returned for user request that wanted ANY acces road type
-    query.accessRoadType = {
-      [Op.eq]: "ANY"
-    };
+  } else if (realEstateTypeObject.hasAccesRoadType) {
+    checkForIncompleteWanted = true;
   }
-  //Tag to check if incomplete ads are accepted in query
+
+  if (balcony) {
+    query.balcony = {
+      [Op.eq]: balcony
+    };
+  } else if (realEstateTypeObject.hasBalconyProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (newBuilding) {
+    query.newBuilding = {
+      [Op.eq]: newBuilding
+    };
+  } else if (realEstateTypeObject.hasNewBuildingProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (elevator) {
+    query.elevator = {
+      [Op.eq]: elevator
+    };
+  } else if (realEstateTypeObject.hasElevatorProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  //If one of the attributes that exists for property type is null
+  //we include in query to check if incomplete real estates are accepted
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
       [Op.eq]: true
     };
   }
-
-  return await db.SearchRequest.findAll({
-    where: query
-  });
+  return await db.SearchRequest.findAll({ where: query });
 };
 
 module.exports = {

app/migrations/20200121000524-add-priceHistory-table.js

@@ -0,0 +1,42 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false,
+        primaryKey: true
+      },
+      realEstateId: {
+        type: Sequelize.BIGINT,
+        allowNull: false,
+        unique: "uniquePriceRealEstate",
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      price: {
+        type: Sequelize.REAL,
+        unique: "uniquePriceRealEstate"
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("PriceHistory", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("PriceHistory", {});
+  }
+};

app/migrations/20200121094500-add-constraint-priceHistory.js

@@ -0,0 +1,10 @@
+"use strict";
+module.exports = {
+  up: (queryInterface, Sequelize) =>
+    queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], {
+      type: "unique",
+      name: "uniquePriceRealEstate"
+    }),
+  down: queryInterface =>
+    queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate")
+};

app/models/priceHistory.js

@@ -0,0 +1,44 @@
+"use strict";
+
+module.exports = (sequalize, DataTypes) => {
+  const PriceHistory = sequalize.define(
+    "PriceHistory",
+    {
+      id: {
+        type: DataTypes.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false
+      },
+      realEstateId: {
+        type: DataTypes.BIGINT,
+        allowNull: false,
+        unique: "uniquePriceRealEstate",
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      price: {
+        type: DataTypes.REAL,
+        unique: "uniquePriceRealEstate"
+      }
+    },
+    {
+      freezeTableName: true
+    }
+  );
+
+  PriceHistory.associate = models => {
+    PriceHistory.hasMany(models.RealEstate, {
+      foreignKey: "id",
+      sourceKey: "realEstateId",
+      targetKey: "id",
+      as: "realEstates"
+    });
+  };
+
+  return PriceHistory;
+};

app/seeders/20200121150443-initial-price-history.js

@@ -0,0 +1,34 @@
+"use strict";
+
+const { RealEstate } = require("../models");
+
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    //Reading initial data from RealEstate table in db
+    const realEstateInitialData = await RealEstate.findAll();
+    //Extruding data for table PriceHistory
+    const priceHistoryInitialData = realEstateInitialData.map(realEstate => {
+      //Null values canot be recognized by ignore duplicates in sequalize
+      //Value price = 0 indicates 'cijena na upit'
+      const priceTmp =
+        realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price;
+
+      return {
+        realEstateId: realEstate.dataValues.id,
+        price: priceTmp,
+        createdAt: realEstate.dataValues.createdAt,
+        updatedAt: realEstate.dataValues.updatedAt
+      };
+    });
+
+    return queryInterface.bulkInsert(
+      "PriceHistory",
+      priceHistoryInitialData,
+      {}
+    );
+  },
+
+  async down(queryInterface, Sequelize) {
+    return queryInterface.bulkDelete("PriceHistory", null, {});
+  }
+};

package.json

@@ -8,6 +8,7 @@
     "start": "node ./index.js",
     "start-mon": "nodemon ./index.js",
     "migrate": "cd app && npx sequelize db:migrate",
+    "seed": "cd app && npx sequelize db:seed:all",
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 10 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",