create new crawler and Postgres saver

app/crawler/crawl.js

@@ -8,48 +8,28 @@
 
 require("dotenv").config();
 const OlxCrawler = require("./specific/olx");
+const { OLX_CONFIG } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
-let crawlers = [
+const crawlers = [
-  // new OlxCrawler(
+  new OlxCrawler(
-  //   process.env.OLX_FROM_PAGE,
+    OLX_CONFIG.OLX_START_PAGE,
-  //   process.env.OLX_TO_PAGE,
+    OLX_CONFIG.OLX_END_PAGE,
-  //   process.env.OLX_MAX_RESULTS
+    OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
-  // )
+    [new PostgresSaver()],
-  // new ProstorCrawler(
+    OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
-  //   parseInt(process.env.PROSTOR_FROM_PAGE),
+    OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES
-  //   parseInt(process.env.PROSTOR_TO_PAGE),
+  )
-  //   parseInt(process.env.PROSTOR_MAX_RESULTS)
-  // ),
-  // new RentalCrawler(
-  //   parseInt(process.env.RENTAL_FROM_PAGE),
-  //   parseInt(process.env.RENTAL_TO_PAGE),
-  //   parseInt(process.env.RENTAL_MAX_RESULTS)
-  // )
 ];
 
-let savers = [new PostgresSaver(process.env.MONGO_URL)];
-
 async function crawlAll() {
   for (let crawler of crawlers) {
     try {
-      const crawlerResults = await crawler.crawl();
+      await crawler.crawl();
-      for (let saver of savers) {
-        try {
-          await saver.connect();
-          await saver.save(crawlerResults);
-        } catch (e) {
-          console.log("Error saving. Trying next saver! ", e);
-        }
-      }
     } catch (e) {
       console.log("Error crawling. Trying next crawler! ", e);
     }
   }
-
-  for (let saver of savers) {
-    saver.close();
-  }
 }
 
 crawlAll();

app/crawler/crawlerConfig.js

@@ -0,0 +1,32 @@
+"use strict";
+require("dotenv").config({ path: "../../.env" });
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../common/enums");
+
+const crawlerAdType =
+  process.env.OLX_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
+    : null;
+
+const parsedCrawlerAdCategories =
+  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["CATEGORY_FLAT", "CATEGORY_HOUSE"];
+
+const transformedCrawlerAdCategories = parsedCrawlerAdCategories
+  .map(categoryName => AD_CATEGORY[categoryName])
+  .filter(category => !!category);
+
+const OLX_CONFIG = {
+  OLX_START_PAGE: parseInt(process.env.OLX_START_PAGE) || 1,
+  OLX_END_PAGE: parseInt(process.env.OLX_END_PAGE) || 10,
+  OLX_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
+  OLX_CRAWLER_AD_TYPE: crawlerAdType || CRAWLER_AD_TYPE.NONE,
+  OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories
+};
+
+module.exports = {
+  OLX_CONFIG
+};

app/crawler/savers/postgres.js

@@ -1,8 +1,6 @@
-class PostgresSaver {
+const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
-  constructor(url) {
-    this.url = url;
-  }
 
+class PostgresSaver {
   connect() {
     //TODO: It seems we never worry about open/close connection with Sequelize ?
     //TODO: Check if postgres is ready
@@ -10,13 +8,8 @@ class PostgresSaver {
   }
 
   async save(results) {
-    let resultsForMongo = Object.keys(results).map(key => {
+    console.log("[POSTGRES] Saving...");
-      return results[key];
+    await bulkUpsertRealEstates(results);
-    });
-
-    for (const doc of resultsForMongo) {
-      this.collection.update({ url: doc.url }, doc, { upsert: true });
-    }
   }
 
   close() {

app/crawler/specific/olx.js

@@ -6,21 +6,136 @@ let cheerio = require("cheerio");
 const {
   AD_TYPE,
   AD_CATEGORY,
-  IGNORED_USERNAMES
+  IGNORED_USERNAMES,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
 } = require("../../common/enums");
 
+const OLX_ENUMS = {
+  OLX_AD_TYPE: {},
+  OLX_AD_CATEGORY: {},
+  MAX_DETAIL_FIELDS: 30
+};
+
+OLX_ENUMS.OLX_AD_TYPE[CRAWLER_AD_TYPE.ALL] = "";
+OLX_ENUMS.OLX_AD_TYPE[CRAWLER_AD_TYPE.ONLY_SELL] = "&vrsta=samoprodaja";
+OLX_ENUMS.OLX_AD_TYPE[CRAWLER_AD_TYPE.ONLY_RENT] = "&vrsta=samoizdavanje";
+
+OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_FLAT] = "&kategorija=23";
+OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_HOUSE] = "&kategorija=24";
+OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_LAND] = "&kategorija=29";
+OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_OFFICE] = "&kategorija=25";
+OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_APARTMENT] = "&kategorija=27";
+OLX_ENUMS.OLX_AD_CATEGORY[AD_CATEGORY.CATEGORY_GARAGE] = "&kategorija=30";
+
 class OlxCrawler {
-  constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
+  constructor(
+    fromPage = 1,
+    toPage = 10,
+    maxResults = 1000,
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [
+      AD_CATEGORY.CATEGORY_FLAT,
+      AD_CATEGORY.CATEGORY_HOUSE
+    ]
+  ) {
     this.fromPage = fromPage;
     this.toPage = toPage;
     this.maxResults = maxResults;
+    this.savers = savers;
+    this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
   }
 
-  async indexSingle(url) {
+  async crawl() {
+    console.log("[OLX] Crawler started");
+    const crawlAdTypes = this.crawlerAdTypes;
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const urlWithAdTypeFilter = `${this.baseUrl}${OLX_ENUMS.OLX_AD_TYPE[crawlAdTypes]}`;
+
+    if (crawlAdCategories && crawlAdTypes) {
+      const asyncPagesIndexingByCategory = [];
+      for (const adCategory of crawlAdCategories) {
+        asyncPagesIndexingByCategory.push(
+          this.indexPages(
+            `${urlWithAdTypeFilter}${OLX_ENUMS.OLX_AD_CATEGORY[adCategory]}`
+          )
+        );
+      }
+
+      await Promise.all(asyncPagesIndexingByCategory);
+    }
+    console.log("[OLX] Crawler finished");
+  }
+
+  async indexPages(url) {
+    const startPage = this.fromPage;
+    const endPage = this.toPage;
+    const maxResultsPerPage = this.maxResults;
+
+    for (let pageNumber = startPage; pageNumber <= endPage; pageNumber++) {
+      const singlePageResults = await this.indexSinglePage(
+        url,
+        pageNumber,
+        maxResultsPerPage
+      );
+      await this.saveCrawledResults(singlePageResults);
+      await this.sleep(5000);
+    }
+  }
+
+  async indexSinglePage(urlWithoutPageNumber, pageNumber, maxResultsPerPage) {
     try {
+      const url = `${urlWithoutPageNumber}&stranica=${pageNumber}`;
+
       const res = await fetch(url);
       const body = await res.text();
       const $ = cheerio.load(body);
+      let hrefs = [];
+      const singlePageResults = [];
+
+      $("#rezultatipretrage")
+        .find(".listitem")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      for (let i = 0; i < actualNoOfResults; i++) {
+        console.log(`Scraping : ${hrefs[i]}`);
+
+        const adData = await this.scrapeAd(hrefs[i]);
+
+        if (adData) {
+          singlePageResults.push(adData);
+        }
+        await this.sleep(500);
+      }
+
+      return singlePageResults;
+    } catch (e) {
+      console.error("Exception caught:" + e);
+    }
+  }
+
+  async scrapeAd(url) {
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+      let status = AD_STATUS.STATUS_NORMAL;
 
       const username = $(
         "#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span"
@@ -31,161 +146,240 @@ class OlxCrawler {
       }
 
       const title = $("#naslovartikla").text();
+      const descriptions = $(".artikal_detaljniopis_tekst");
       const category = $(
         "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span"
       ).text();
 
-      const price = $("#pc > p:nth-child(2)").text();
+      //====== PRICE DETECTION AND EXTRACTION =====
-      const size = $("#dodatnapolja1 > div:nth-child(1) > div.df2").text();
+      let price = null;
-      const rooms = $("#dodatnapolja1 > div:nth-child(2) > div.df2").text();
+      const normalPriceValue = $("#pc > p:nth-child(2)").text();
-      const address = $("#dodatnapolja1 > div:nth-child(5) > div.df2").text();
+      const urgentPriceValue = $(
-      const location = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
-        "#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija"
-      ).attr("data-content");
-
-      const adType = $(
-        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2"
-      ).text();
-      const time = $("time").attr("datetime");
-      const olxId = $(
-        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2"
       ).text();
 
-      const descriptions = $(".artikal_detaljniopis_tekst");
+      if (normalPriceValue && normalPriceValue.length > 0) {
-      const floor = $("#dodatnapolja1")
+        price = normalPriceValue;
-        .find(":contains(Sprat)")
+        if (
-        .last()
+          $("#pc > p.n")
-        .nextAll()
+            .text()
-        .text();
+            .indexOf("Hitna") !== -1
-      const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+        ) {
-      const matches = latLngRe.exec(body);
+          status = AD_STATUS.STATUS_URGENT;
-      let lng = "",
+        } else {
-        lat = "";
+          status = AD_STATUS.STATUS_NORMAL;
-
+        }
-      const parseRooms = rooms =>
+      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
-        parseInt(
+        const priceValues = urgentPriceValue.split("KM");
-          [...rooms]
+        //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
-            .filter(c => !isNaN(c))
+        if (priceValues.length > 1) {
-            .filter(c => c.trim())
+          price = priceValues[1].trim();
-            .join()
+          status = AD_STATUS.STATUS_DISCOUNTED;
-        );
+        } else {
-      const parsePrice = price => parseFloat(price.replace(".", ""));
+          throw { message: "Can't find urgent price" };
-
+        }
-      if (matches && matches.length >= 3) {
+      } else {
-        lat = matches[1];
+        throw {
-        lng = matches[2];
+          message: "Can't find price (it is not normal nor urgent price ?)"
+        };
       }
 
-      const parsedPrice = parsePrice(price);
+      //====== OTHER AD INFORMATION ===============
-      let parsedRooms;
+      let adType = null;
+      let olxId = null;
 
-      if (rooms === "Garsonjera") {
+      let otherInformationDivId;
-        parsedRooms = 0;
+      //We need to locate DIV ID where other information are stored
-      } else {
+      for (let possibleId = 10; possibleId <= 20; possibleId++) {
-        parsedRooms = parseRooms(rooms);
+        const adTypeFieldTitle = $(
+          `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
+        )
+          .text()
+          .trim();
+
+        if (adTypeFieldTitle === "Vrsta oglasa") {
+          otherInformationDivId = possibleId;
+          break;
+        }
+      }
+
+      if (!otherInformationDivId) {
+        throw { message: "Other information DIV could not be found" };
+      }
+
+      const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
+
+      adType = $(
+        `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
+      )
+        .text()
+        .trim();
+      const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
+        .text()
+        .trim();
+      olxId = $(`${olxIdFieldSelector} > div.df2`)
+        .text()
+        .trim();
+
+      if (olxIdFieldTitle !== "OLX ID") {
+        throw { message: "Cannot find correct OLX ID" };
+      }
+      //===========================================
+
+      //====== DETAIL INFORMATION FIELDS ==========
+      let area = null;
+      let gardenSize = null;
+
+      let fieldIndex = 1;
+      do {
+        const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
+        const fieldTitleSelector = `${fieldSelector} > div.df1`;
+        const fieldValueSelector = `${fieldSelector} > div.df2`;
+
+        const fieldTitle = $(fieldTitleSelector)
+          .text()
+          .trim();
+        const fieldValue = $(fieldValueSelector)
+          .text()
+          .trim();
+
+        switch (fieldTitle) {
+          case "Kvadrata":
+            area = fieldValue;
+            break;
+          case "Okućnica (kvadratura)":
+            gardenSize = fieldValue;
+            break;
+        }
+
+        if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
+          break;
+        }
+      } while (true);
+      //===========================================
+
+      //====== UNUSED FIELDS FOR NOW ==============
+      const time = $("time").attr("datetime");
+      const numberOfViews = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(18) > div:nth-child(6) > div.df2"
+      ).text();
+      //===========================================
+
+      //=========================================
+      const parsedCategory = this.getAdCategoryId(category);
+      if (!parsedCategory) {
+        throw { message: "Unknown ad category" };
+      }
+
+      const parsedAdType = this.getAdTypeId(adType);
+      if (!parsedAdType) {
+        throw { message: "Unknown ad type" };
+      }
+
+      const parsedArea = this.parseArea(area) || null;
+      const parsedGardenSize = this.parseArea(gardenSize) || null;
+      const parsedPrice = this.parsePrice(price) || null;
+
+      const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+      const locationLatLngMatches = latLngRegex.exec(body);
+
+      let locationLat = null;
+      let locationLong = null;
+      if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
+        locationLat = parseFloat(locationLatLngMatches[1]) || null;
+        locationLong = parseFloat(locationLatLngMatches[2]) || null;
       }
 
       const data = {
-        category: this.getCategoryId(category),
         url,
+        agencyObjectId: olxId,
+        originAgencyName: AD_AGENCY.OLX,
+        realEstateType: this.getAdCategoryId(category),
+        adType: parsedAdType,
         title,
-        price: isNaN(parsedPrice) ? price : parsedPrice,
+        price: parsedPrice,
-        size: parseFloat(size),
+        area: parsedArea,
-        rooms: parsedRooms,
+        gardenSize: parsedGardenSize,
-        floor: parseInt(floor),
-        address,
-        location,
-        adType: AD_TYPE.AD_TYPE_SALE,
-        time,
         shortDescription: descriptions.first().text(),
         longDescription: descriptions.last().text(),
-        lat,
+        streetNumber: 0,
-        lng,
+        streetName: "",
-        loc: [parseFloat(lat), parseFloat(lng)]
+        locality: "",
+        municipality: "",
+        city: "",
+        region: "",
+        entity: "",
+        country: "",
+        locationLat,
+        locationLong,
+        adStatus: status
       };
 
       return data;
     } catch (e) {
-      console.error("Exception caught: " + e.message);
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
     }
-
     return null;
   }
 
-  async indexPage(pageNr, maxResults = 1000) {
+  //======= HELPER FUNCTIONS =============
-    try {
-      console.log("Starting to index page: " + pageNr);
-      const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
 
-      const res = await fetch(url);
+  getAdCategoryId(categoryText) {
-      const body = await res.text();
+    switch (categoryText) {
-      const $ = cheerio.load(body);
+      case "Stanovi":
-      const hrefs = [];
+        return AD_CATEGORY.CATEGORY_FLAT;
-      const results = {};
+      case "Zemljišta":
-
+        return AD_CATEGORY.CATEGORY_LAND;
-      $("#rezultatipretrage")
+      case "Kuće":
-        .find(".listitem")
+        return AD_CATEGORY.CATEGORY_HOUSE;
-        .each((i, elem) => {
+      case "Poslovni prostori":
-          const href = $(elem)
+        return AD_CATEGORY.CATEGORY_OFFICE;
-            .find("a")
+      default:
-            .first()
+        return undefined;
-            .attr("href");
-          hrefs.push(href);
-        });
-
-      let actualNoOfResults =
-        hrefs.length <= maxResults ? hrefs.length : maxResults;
-
-      for (let i = 0; i < hrefs.length; i++) {
-        console.log(`indexing: ${hrefs[i]}`);
-
-        const singleData = await this.indexSingle(hrefs[i]);
-
-        if (singleData) {
-          results[hrefs[i]] = singleData;
-        }
-        await this.sleep(500);
-      }
-
-      return results;
-    } catch (e) {
-      console.error("Exception caught:" + e);
     }
   }
 
-  getCategoryId(category) {
+  getAdTypeId(adTypeText) {
-    if (category === "Stanovi") {
+    switch (adTypeText) {
-      return AD_CATEGORY.CATEGORY_FLAT;
+      case "Prodaja":
-    } else if (category === "Zemljišta") {
+        return AD_TYPE.AD_TYPE_SALE;
-      return AD_CATEGORY.CATEGORY_LAND;
+      case "Izdavanje":
-    } else if (category === "Kuće") {
+        return AD_TYPE.AD_TYPE_RENT;
-      return AD_CATEGORY.CATEGORY_HOUSE;
+      default:
-    } else if (category === "Poslovni prostori") {
+        return undefined;
-      return AD_CATEGORY.CATEGORY_OFFICE;
     }
   }
 
+  parseArea(areaText) {
+    if (!areaText) {
+      return NaN;
+    }
+    const removeDotsExceptLastOneRegex = /[.](?=.*[.])/g;
+    const textWithOnlyOneDecimalDot = areaText
+      .replace(",", ".")
+      .replace(removeDotsExceptLastOneRegex, "");
+
+    return parseFloat(textWithOnlyOneDecimalDot);
+  }
+
+  parsePrice(priceText) {
+    if (!priceText) {
+      return NaN;
+    }
+    const formattedPriceText = priceText.replace(".", "").replace(",", ".");
+    return parseFloat(formattedPriceText);
+  }
+
   async sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
   }
 
-  async indexPages(start, end, maxResults = 1000) {
+  async saveCrawledResults(results) {
-    let results = {};
+    const savers = this.savers;
-    for (let i = start; i <= end; i++) {
-      let result = await this.indexPage(i, maxResults);
-      Object.assign(results, result);
-      await this.sleep(5000);
-    }
-    return results;
-  }
 
-  async crawl() {
+    for (const saver of savers) {
-    let results = await this.indexPages(
+      await saver.save(results);
-      this.fromPage,
+    }
-      this.toPage,
-      this.maxResults
-    );
-    return results;
   }
 }