old-web/app/crawler/specific/olx.js

"use strict";

const fetch = require("node-fetch");
const cheerio = require("cheerio");
const Promise = require("bluebird");
const moment = require("moment-timezone");

const {
  AD_TYPE,
  AD_CATEGORY,
  IGNORED_USERNAMES,
  AD_AGENCY,
  AD_STATUS,
  CRAWLER_AD_TYPE
} = require("../../common/enums");

const { DEFAULT_TIMEZONE } = require("../../config/appConfig");

const OLX_ENUMS = {
  OLX_AD_TYPE: {
    [CRAWLER_AD_TYPE.ALL]: "",
    [CRAWLER_AD_TYPE.ONLY_SELL]: "&vrsta=samoprodaja",
    [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje"
  },
  OLX_AD_CATEGORY: {
    [AD_CATEGORY.CATEGORY_FLAT]: "&kategorija=23",
    [AD_CATEGORY.CATEGORY_HOUSE]: "&kategorija=24",
    [AD_CATEGORY.CATEGORY_LAND]: "&kategorija=29",
    [AD_CATEGORY.CATEGORY_OFFICE]: "&kategorija=25",
    [AD_CATEGORY.CATEGORY_APARTMENT]: "&kategorija=27",
    [AD_CATEGORY.CATEGORY_GARAGE]: "&kategorija=30"
  },
  MAX_DETAIL_FIELDS: 30,
  OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
  OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
};

class OlxCrawler {
  constructor(
    savers = [],
    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
    crawlerAdCategories = [
      AD_CATEGORY.CATEGORY_FLAT,
      AD_CATEGORY.CATEGORY_HOUSE
    ],
    maxPages = 1000,
    maxResultsPerPage = 100,
    maxAge = 30
  ) {
    this.savers = savers;
    this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
    this.crawlerAdTypes = crawlerAdTypes;
    this.crawlerAdCategories = crawlerAdCategories;
    this.maxPages = maxPages;
    this.maxResultsPerPage = maxResultsPerPage;
    this.maxAge = maxAge;
  }

  async crawl() {
    console.log("[OLX] Crawler started");
    const crawlAdCategories = this.crawlerAdCategories;

    if (crawlAdCategories) {
      const indexGenerators = [];
      for (const adCategory of crawlAdCategories) {
        indexGenerators.push(this.categoryIndexer(adCategory));
      }

      let done = false;
      while (!done) {
        const categoryIndexerPromises = [];
        for (const indexGenerator of indexGenerators) {
          categoryIndexerPromises.push(indexGenerator.next());
        }

        Promise.all(categoryIndexerPromises).then(singlePageResults => {
          const entries = singlePageResults.entries();
          for (const [index, { value: singlePageResult }] of entries) {
            if (singlePageResult) {
              this.saveCrawledResults(singlePageResult, this.maxAge)
                .then(numberOfSaved => {})
                .catch(error =>
                  console.log("[POSTGRES Saver] Error saving results : ", error)
                );
            } else {
              //Generator returned undefined, no more pages
              indexGenerators.splice(index, 1);
              if (indexGenerators.length === 0) {
                done = true;
              }
            }
          }
        });

        await this.sleep(500);
      }
    }

    console.log("[OLX] Crawler finished");
  }

  async *categoryIndexer(adCategory) {
    let pageToIndex = 1;

    const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
    const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
    if (urlAdTypePart && urlCategoryPart) {
      while (true) {
        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
        const singlePageResults = await this.indexSinglePage(
          urlPageToCrawl,
          this.maxResultsPerPage
        );
        console.log("indexing ", adCategory, " page : ", pageToIndex);

        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
          yield singlePageResults;
        } else {
          return undefined;
        }

        ++pageToIndex;
        if (pageToIndex === this.maxPages) {
          return undefined;
        }
      }
    } else {
      return undefined;
    }
  }

  async indexSinglePage(url, maxResultsPerPage) {
    try {
      const res = await fetch(url);
      const body = await res.text();
      const $ = cheerio.load(body);
      let hrefs = [];
      const singlePageResults = [];

      $("#rezultatipretrage")
        .find(".listitem")
        .each((i, elem) => {
          const href = $(elem)
            .find("a")
            .first()
            .attr("href");
          if (href) {
            hrefs.push(href);
          }
        });

      let actualNoOfResults =
        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;

      const asyncScraping = [];
      for (let i = 0; i < actualNoOfResults; i++) {
        asyncScraping.push(this.scrapeAd(hrefs[i]));
      }

      const scrapedData = await Promise.all(asyncScraping);
      return scrapedData;
    } catch (e) {
      console.error("Exception caught:" + e);
      return [];
    }
  }

  async scrapeAd(url) {
    try {
      const adPageSource = await fetch(url);
      const body = await adPageSource.text();
      const $ = cheerio.load(body);
      let status = AD_STATUS.STATUS_NORMAL;

      const username = $(
        "#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span"
      )
        .text()
        .trim();

      if (IGNORED_USERNAMES.includes((username || "").toLowerCase())) {
        return null;
      }

      const title = $("#naslovartikla")
        .text()
        .trim();
      const descriptions = $(".artikal_detaljniopis_tekst");
      const category = $(
        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span"
      )
        .text()
        .trim();

      //====== PRICE DETECTION AND EXTRACTION =====
      let price = null;
      const normalPriceValue = $("#pc > p:nth-child(2)").text();
      const urgentPriceValue = $(
        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
      )
        .text()
        .trim();

      if (normalPriceValue && normalPriceValue.length > 0) {
        price = normalPriceValue;
        if (
          $("#pc > p.n")
            .text()
            .indexOf("Hitna") !== -1
        ) {
          status = AD_STATUS.STATUS_URGENT;
        } else {
          status = AD_STATUS.STATUS_NORMAL;
        }
      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
        const priceValues = urgentPriceValue.split("KM");
        //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
        if (priceValues.length > 1) {
          price = priceValues[1].trim();
          status = AD_STATUS.STATUS_DISCOUNTED;
        } else {
          throw { message: "Can't find urgent price" };
        }
      } else {
        throw {
          message: "Can't find price (it is not normal nor urgent price ?)"
        };
      }

      //====== OTHER AD INFORMATION ===============
      let adType = null;
      let olxId = null;

      let otherInformationDivId;
      //We need to locate DIV ID where other information are stored
      for (let possibleId = 10; possibleId <= 20; possibleId++) {
        const adTypeFieldTitle = $(
          `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
        )
          .text()
          .trim();

        if (adTypeFieldTitle === "Vrsta oglasa") {
          otherInformationDivId = possibleId;
          break;
        }
      }

      if (!otherInformationDivId) {
        throw { message: "Other information DIV could not be found" };
      }

      const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
      const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
      const renewedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(5) > div.df2`;

      const publishedDate = $(publishedDateValueSelector)
        .text()
        .trim();

      const publishedDateMoment = moment.tz(
        publishedDate,
        OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
        DEFAULT_TIMEZONE
      );

      if (!publishedDateMoment.isValid()) {
        throw { message: "Invalid published date ! Check parsing format" };
      }

      const renewedDate = $(renewedDateValueSelector)
        .text()
        .trim();

      const renewedDateMoment = this.parseRenewedDate(renewedDate);

      if (!renewedDateMoment) {
        throw {
          message:
            "Invalid renewed date ! Check how parser parsed renewed date text"
        };
      }

      adType = $(
        `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
      )
        .text()
        .trim();
      const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
        .text()
        .trim();
      olxId = $(`${olxIdFieldSelector} > div.df2`)
        .text()
        .trim();

      if (olxIdFieldTitle !== "OLX ID") {
        throw { message: "Cannot find correct OLX ID" };
      }
      //===========================================

      //====== DETAIL INFORMATION FIELDS ==========
      let area = null;
      let gardenSize = null;

      let fieldIndex = 1;
      do {
        const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
        const fieldTitleSelector = `${fieldSelector} > div.df1`;
        const fieldValueSelector = `${fieldSelector} > div.df2`;

        const fieldTitle = $(fieldTitleSelector)
          .text()
          .trim();
        const fieldValue = $(fieldValueSelector)
          .text()
          .trim();

        switch (fieldTitle) {
          case "Kvadrata":
            area = fieldValue;
            break;
          case "Okućnica (kvadratura)":
            gardenSize = fieldValue;
            break;
        }

        if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
          break;
        }
      } while (true);
      //===========================================

      //====== UNUSED FIELDS FOR NOW ==============
      const time = $("time").attr("datetime");
      const numberOfViews = $(
        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(18) > div:nth-child(6) > div.df2"
      )
        .text()
        .trim();
      //===========================================

      //=========================================
      const parsedCategory = this.getAdCategoryId(category);
      if (!parsedCategory) {
        throw { message: "Unknown ad category" };
      }

      const parsedAdType = this.getAdTypeId(adType);
      if (!parsedAdType) {
        throw { message: "Unknown ad type" };
      }

      const parsedArea = this.parseArea(area) || null;
      const parsedGardenSize = this.parseArea(gardenSize) || null;
      const parsedPrice = this.parsePrice(price) || null;

      const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
      const locationLatLngMatches = latLngRegex.exec(body);

      let locationLat = null;
      let locationLong = null;
      if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
        locationLat = parseFloat(locationLatLngMatches[1]) || null;
        locationLong = parseFloat(locationLatLngMatches[2]) || null;
      }

      const data = {
        url,
        agencyObjectId: olxId,
        originAgencyName: AD_AGENCY.OLX,
        realEstateType: this.getAdCategoryId(category),
        adType: parsedAdType,
        title,
        price: parsedPrice,
        area: parsedArea,
        gardenSize: parsedGardenSize,
        shortDescription: descriptions
          .first()
          .text()
          .trim(),
        longDescription: descriptions
          .last()
          .text()
          .trim(),
        streetNumber: 0,
        streetName: "",
        locality: "",
        municipality: "",
        city: "",
        region: "",
        entity: "",
        country: "",
        locationLat,
        locationLong,
        adStatus: status,
        publishedDate: publishedDateMoment.toISOString(),
        renewedDate: renewedDateMoment.toISOString()
      };

      return data;
    } catch (e) {
      console.error("Exception caught: " + e.message, "\r\nURL:", url);
    }
    return null;
  }

  //======= HELPER FUNCTIONS =============

  getAdCategoryId(categoryText) {
    switch (categoryText) {
      case "Stanovi":
        return AD_CATEGORY.CATEGORY_FLAT;
      case "Zemljišta":
        return AD_CATEGORY.CATEGORY_LAND;
      case "Kuće":
        return AD_CATEGORY.CATEGORY_HOUSE;
      case "Poslovni prostori":
        return AD_CATEGORY.CATEGORY_OFFICE;
      default:
        return undefined;
    }
  }

  getAdTypeId(adTypeText) {
    switch (adTypeText) {
      case "Prodaja":
        return AD_TYPE.AD_TYPE_SALE;
      case "Izdavanje":
        return AD_TYPE.AD_TYPE_RENT;
      default:
        return undefined;
    }
  }

  parseArea(areaText) {
    if (!areaText) {
      return NaN;
    }
    const removeDotsExceptLastOneRegex = /[.](?=.*[.])/g;
    const textWithOnlyOneDecimalDot = areaText
      .replace(",", ".")
      .replace(removeDotsExceptLastOneRegex, "");

    return parseFloat(textWithOnlyOneDecimalDot);
  }

  parsePrice(priceText) {
    if (!priceText) {
      return NaN;
    }
    const formattedPriceText = priceText.replace(".", "").replace(",", ".");
    return parseFloat(formattedPriceText);
  }

  parseRenewedDate(renewedDateText) {
    const currentMoment = moment.tz(DEFAULT_TIMEZONE);

    if (renewedDateText.includes("Prije mjesec dana")) {
      return currentMoment.add(-1, "month");
    }

    const dayVariations = ["dan", "dana"];
    for (const dayVariation of dayVariations) {
      if (renewedDateText.includes(dayVariation)) {
        // format for this case should be "Prije N dana" or "Prije N dan"
        const dateParts = renewedDateText.split(" ");
        if (dateParts[0] === "Prije") {
          const numberOfDays = parseInt(dateParts[1]);
          return currentMoment.add(-1 * numberOfDays, "days");
        } else {
          return undefined;
        }
      }
    }

    if (renewedDateText.includes("Jučer")) {
      return currentMoment.add(-1, "day");
    }

    const todayVariations = [
      "sat",
      "sati",
      "sata",
      "min",
      "sekunde",
      "sekundi",
      "sekundu",
      "maloprije"
    ];
    for (const todayVariation of todayVariations) {
      if (renewedDateText.includes(todayVariation)) {
        return currentMoment;
      }
    }

    const renewedDateMoment = moment.tz(
      renewedDateText,
      OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
      DEFAULT_TIMEZONE
    );

    return renewedDateMoment.isValid() ? renewedDateMoment : undefined;
  }

  async sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  async saveCrawledResults(results, maxAge) {
    const savers = this.savers;

    for (const saver of savers) {
      await saver.save(results, maxAge);
    }
  }
}

module.exports = OlxCrawler;