Merge branch 'add-crawler-for-rental-page' into 'master'

Add crawler for rental page

See merge request saburly/marketalarm/web!58

app/common/enums.js

@@ -133,7 +133,8 @@ const AD_STATUS = {
 };
 
 const AD_AGENCY = {
-  OLX: "OLX"
+  OLX: "OLX",
+  RENTAL: "RENTAL"
 };
 
 const CRAWLER_AD_TYPE = {

app/crawler/crawl.js

@@ -6,30 +6,49 @@
    passed to the crawlers and savers.
 */
 const OlxCrawler = require("./specific/olx");
-const { OLX_CONFIG } = require("./crawlerConfig");
+const RentalCrawler = require("./specific/rental");
+
+const { OLX_CONFIG, RENTAL_CONFIG } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
-const crawlers = [
-  new OlxCrawler(
-    [new PostgresSaver()],
-    OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
-    OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
-    OLX_CONFIG.OLX_MAX_PAGES,
-    OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
-    OLX_CONFIG.OLX_IGNORED_USERNAMES,
-    OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
-  )
-];
-
 async function crawlAll() {
-  for (let crawler of crawlers) {
+  const postgresSaver = new PostgresSaver();
+
+  const crawlers = [
+    new OlxCrawler(
+      [postgresSaver],
+      OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
+      OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
+      OLX_CONFIG.OLX_MAX_PAGES,
+      OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
+      OLX_CONFIG.OLX_IGNORED_USERNAMES,
+      OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
+    ),
+    new RentalCrawler(
+      [postgresSaver],
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
+      RENTAL_CONFIG.RENTAL_MAX_PAGES,
+      RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
+      RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
+      RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
+    )
+  ];
+
+  const newRealEstates = [];
+
+  for (const crawler of crawlers) {
     try {
-      return await crawler.crawl();
+      const newRealEstatesFromSingleCrawler = await crawler.crawl();
+      if (Array.isArray(newRealEstatesFromSingleCrawler)) {
+        newRealEstates.push(...newRealEstatesFromSingleCrawler);
+      }
     } catch (e) {
       console.log("Error crawling. Trying next crawler! ", e);
-      return [];
     }
   }
+
+  return newRealEstates;
 }
 
 module.exports = {

app/crawler/crawlerConfig.js

@@ -7,6 +7,11 @@ const olxCrawlerAdType =
     ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
     : null;
 
+const rentalCrawlerAdType =
+  process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
+    : null;
+
 const olxParsedCrawlerAdCategories =
   process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
     ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
@@ -14,6 +19,13 @@ const olxParsedCrawlerAdCategories =
       )
     : ["FLAT", "HOUSE"];
 
+const rentalParsedCrawlerAdCategories =
+  process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
 const olxIgnoredUsernames =
   process.env.OLX_IGNORED_USERNAMES !== undefined
     ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
@@ -21,7 +33,15 @@ const olxIgnoredUsernames =
       )
     : [];
 
-const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories
+const rentalIgnoredUsernames = [];
+
+const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
   .map(categoryName =>
     AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
   )
@@ -32,11 +52,23 @@ const OLX_CONFIG = {
   OLX_MAX_RESULTS_PER_PAGE:
     parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
   OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
-  OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories,
+  OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
   OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
   OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
 };
 
-module.exports = {
+const RENTAL_CONFIG = {
-  OLX_CONFIG
+  RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
+  RENTAL_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
+  RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
+  RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
+  RENTAL_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000
+};
+
+module.exports = {
+  OLX_CONFIG,
+  RENTAL_CONFIG
 };

app/crawler/specific/rental.js

@@ -0,0 +1,376 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+const htmlToText = require("html-to-text");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const { DEFAULT_TIMEZONE } = require("../../config/appConfig");
+
+const RENTAL_ENUMS = {
+  RENTAL_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
+  },
+  RENTAL_AD_CATEGORY: {
+    [AD_CATEGORY.FLAT.id]: "/tip-2",
+    [AD_CATEGORY.HOUSE.id]: "/tip-1",
+    [AD_CATEGORY.LAND.id]: "/tip-5",
+    [AD_CATEGORY.OFFICE.id]: "/tip-4",
+    [AD_CATEGORY.APARTMENT.id]: "/tip-3",
+    [AD_CATEGORY.GARAGE.id]: "/tip-6"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  RENTAL_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+class RentalCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.rental.ba/pretraga/sortiraj-date_DESC";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (Array.isArray(newRecords) && newRecords.length === 0) {
+              generatorsToRemove[index] = true;
+            }
+
+            // for (const existingRecord of existingRecords) {
+            //   const { publishedDate, renewedDate } = existingRecord;
+            //
+            //   const publishedDateMoment = moment.utc(publishedDate);
+            //   const renewedDateMoment = moment.utc(renewedDate);
+            //
+            //   const stopCrawlingThisCategory = publishedDateMoment.isSame(
+            //     renewedDateMoment,
+            //     "minute"
+            //   );
+            //
+            //   if (stopCrawlingThisCategory) {
+            //     generatorsToRemove[index] = true;
+            //     // console.log("\tGenerator ", index + 1, "has no more new ads");
+            //     break;
+            //   }
+            // }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
+    if (urlAdTypePart && urlCategoryPart) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    // console.log("[RENTAL] Index page : ", url);
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $(
+        "body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
+      )
+        .find(".pull-right")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[RENTAL] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("[RENTAL] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      const mapElementParent = $(".box-map").parent();
+      const scriptElement = $("script", mapElementParent);
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        let extractedData;
+        try {
+          //data string starts with : var json_map_data = [{"r ...
+          //so we remove first 20 characters
+
+          const jsonData = scriptElement[0].children[0].data.substring(20);
+          const parsedJsonData = JSON.parse(jsonData);
+          extractedData = parsedJsonData[0];
+        } catch (e) {
+          throw { message: "Can't find ad data JSON" };
+        }
+
+        const rentalId = extractedData["re_realEstates_id"];
+        const adCategory = this.getKiviCategoryIdFromRentalId(
+          parseInt(extractedData["re_types_id"])
+        );
+        if (!adCategory) {
+          throw {
+            message: `Invalid category : ${extractedData["re_types_id"]}`
+          };
+        }
+        const adType = this.getKiviAdTypeFromRentalActionId(
+          parseInt(extractedData["re_action_id"])
+        );
+        if (!adType) {
+          throw {
+            message: `Invalid ad type : ${extractedData["re_action_id"]}`
+          };
+        }
+
+        const title = extractedData["re_realEstates_portalName"];
+        const extractedPrice = parseFloat(
+          extractedData["re_realEstates_price"]
+        );
+        const price = extractedPrice ? extractedPrice : null;
+        const area = parseFloat(extractedData["re_realEstates_area"]);
+        const gardenSize = parseFloat(
+          extractedData["re_realEstates_fieldArea"]
+        );
+        const longDescription = htmlToText.fromString(
+          extractedData["re_realEstates_description"]
+        );
+        const locationLong = extractedData["re_realEstates_longitude"];
+        const locationLat = extractedData["re_realEstates_latitude"];
+        const publishedDateMoment = moment.tz(
+          extractedData["re_realEstates_inserted"],
+          RENTAL_ENUMS.RENTAL_PUBLISHED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!publishedDateMoment.isValid()) {
+          throw {
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
+          };
+        }
+
+        const renewedDateMoment = moment.tz(
+          extractedData["re_realEstates_edited"],
+          RENTAL_ENUMS.RENTAL_RENEWED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!renewedDateMoment.isValid()) {
+          throw {
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
+          };
+        }
+
+        const adStatus = AD_STATUS.STATUS_NORMAL;
+
+        const data = {
+          url,
+          agencyObjectId: rentalId,
+          originAgencyName: AD_AGENCY.RENTAL,
+          realEstateType: adCategory,
+          adType,
+          title,
+          price,
+          area,
+          gardenSize,
+          shortDescription: "",
+          longDescription: longDescription,
+          streetNumber: 0,
+          streetName: "",
+          locality: "",
+          municipality: "",
+          city: "",
+          region: "",
+          entity: "",
+          country: "",
+          locationLat,
+          locationLong,
+          adStatus,
+          publishedDate: publishedDateMoment.toISOString(),
+          renewedDate: renewedDateMoment.toISOString()
+        };
+
+        return data;
+      } else {
+        console.log("[RENTAL] No JSON data for this ad : ", url);
+        return null;
+      }
+    } catch (e) {
+      console.error("[RENTAL] Exception caught: " + e.message, "\r\nURL:", url);
+      return null;
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getKiviCategoryIdFromRentalId(rentalCategoryId) {
+    switch (rentalCategoryId) {
+      case 1:
+        return AD_CATEGORY.HOUSE.id;
+      case 2:
+        return AD_CATEGORY.FLAT.id;
+      case 3:
+        return AD_CATEGORY.APARTMENT.id;
+      case 4:
+        return AD_CATEGORY.OFFICE.id;
+      case 5:
+        return AD_CATEGORY.LAND.id;
+      case 6:
+        return AD_CATEGORY.GARAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getKiviAdTypeFromRentalActionId(actionId) {
+    switch (actionId) {
+      case 1:
+        return AD_TYPE.AD_TYPE_SALE;
+      case 2:
+        return AD_TYPE.AD_TYPE_RENT;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = RentalCrawler;

development.env

@@ -30,3 +30,10 @@ OLX_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check commo
 OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
 OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
 OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
+#==RENTAL==
+RENTAL_MAX_PAGES=Restrict crawler to this number of pages
+RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
+RENTAL_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
+RENTAL_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
+RENTAL_IGNORED_USERNAMES=!!! This is not used for rental crawler !!!
+RENTAL_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page