old-web/app/helpers/crawlers/olxClawler.js

const fetch = require('node-fetch');
const cheerio = require('cheerio');
const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
const { getRealEstateTypeEnum } = require('../enums');
const { getRegion, getMunicipality } = require('../codes')
const Promise = require("bluebird");

module.exports = class OlxCrawler {
    //TODO figure best way to handle paging
    constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
        this.fromPage = fromPage;
        this.toPage = toPage;
        this.maxResults = maxResults;
    }

    async indexSingle(url, email, uuid) {
        try {
            const res = await fetch(url);
            const body = await res.text();
            const $ = cheerio.load(body);

            //TODO figure out what to do with username
            const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();

            // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
            //     return null;
            // }

            //TODO remove properties that are not needed, and add some if they are missing
            const title = $('#naslovartikla').text().trim();
            const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();

            const price = $('#pc > p:nth-child(2)').text();
            const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
            const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
            const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
            const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
            const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');

            const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
            const time = $('time').attr('datetime');
            const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();

            const descriptions = $('.artikal_detaljniopis_tekst');
            // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
            const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
            const imgRe = /href":("[^"]*")/g;
            const matches = latLngRe.exec(body);
            let lng = '',
                lat = '';


            const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
            const parsePrice = (price) => parseFloat(price.replace(".", ""))


            // TODO we dont save images ??

            // const images = [];
            // const imgMatches = body.match(imgRe);

            // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
            //     let img = imgMatches[i].replace("href\":", "")
            //     img = img.replace("\"", "");
            //     img = img.replace("\"", "");
            //     images.push(img);
            // }

            // const uploadPromises = images.map(img => {
            //     const imgFixed = eval(`'${img}'`);
            //     return cloudinary.uploader.upload(eval(`'${img}'`));
            // });

            // const uploadResults = await Promise.all(uploadPromises);
            // const cloudinaryImages = uploadResults.map(ur => ur.url);

            if (matches && matches.length >= 3) {
                lat = matches[1];
                lng = matches[2];
            }

            const parsedPrice = parsePrice(price);

            console.log(location);
            const locationArray = location.split(",");
            const region = locationArray[0];
            const municipality = locationArray[1];

            const data = {
                realEstateType: this.getCategoryId(realEstateType),
                email: email,
                uuid: uuid,
                olxId: olxId,
                // category: category,
                url,
                title,
                price: isNaN(parsedPrice) ? 0 : parsedPrice,
                size: parseFloat(size),
                gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
                address,
                region,
                municipality,
                // adType: AD_TYPE_SALE,
                time,
                shortDescription: descriptions.first().text(),
                longDescription: descriptions.last().text(),
                lat,
                lng,
                loc: [parseFloat(lat), parseFloat(lng)],
                // images: cloudinaryImages
            };

            return data;
        } catch (e) {
            console.error('Exception caught: ' + e.message);
        }

        return null;
    }

    async indexPage(olxUrl, maxResults = 1000) {
        try {
            //TODO fix paging
            // console.log('Starting to index page: ' + pageNr);
            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;

            const res = await fetch(olxUrl.url);
            const body = await res.text();
            const $ = cheerio.load(body);
            const hrefs = [];
            const results = [];

            $('#rezultatipretrage').find('.listitem').each((i, elem) => {
                const href = $(elem).find('a').first().attr('href');
                hrefs.push(href);
            });

            let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;

            for (let i = 0; i < hrefs.length; i++) {
                console.log(`indexing: ${hrefs[i]}`);

                const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid);

                if (singleData) {
                    results.push(singleData);
                }
            }

            return results;
        } catch (e) {
            console.error('Exception caught:' + e);
        }
    }

    getCategoryId(category) {

        switch (category) {
            case 'Stanovi':
                return 'stan';

            case 'Vikendice':
                return 'vikendica'

            case 'Kuće':
                return 'kuca';

            default:
                return '';
        }
    }

    async indexPages(urls, start, end, maxResults = 1000) {
        //TODO fix paging
        // let results = {};
        // for (let i = start; i <= end; i++) {
        //     let result = await this.indexPage(i, maxResults);
        //     Object.assign(results, result)
        // }
        // return results;

        // let results = [];
        const indexers= [];
        // let it = 3
        // for (let url of urls) {
        //     // let result = await this.indexPage(url, maxResults);
        //     // results.push(result);
        //     it++
        //     indexers.push(new Indexer(url));

        // }

        urls.forEach(url => {
            indexers.push(new Indexer(url));
        });

        return Promise.map(indexers, function (indexer) {
            return indexer.indexWithPagination();
        }).then(async (results) => {
            return results
        })       


        // let results = [];
        // for (let url of urls) {
        //     let result = await this.indexPage(url, maxResults);
        //     results.push(result);
        // }
        // return results;
    }

    async crawl() {
        console.log("OLX CRAWLER: start crawl");

        const filteredResults = [];
        const realestateRequests = await allRERequest();
        console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
        const urls = this.createRequestUrls(realestateRequests);
        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);

        console.log(results[0]);

        // for (const result of results) {
        //     for (const finalResult of result) {
        //         if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
        //             const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);

        //             if (pointInsideBoundingBox[0].length !== 0) {
        //                 filteredResults.push(finalResult);
        //             }
        //         }
        //     }
        // }
        // console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
        // return filteredResults;
    }

    createRequestUrls(realestateRequests) {
        const urls = []

        for (const request of realestateRequests) {
            const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
            const region = "kanton=" + getRegion(request.region).olxid;
            const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
            const sizeMin = "kvadrata_min=" + request.sizeMin;
            const sizeMax = "kvadrata_max=" + request.sizeMax;
            const priceMin = "od=" + request.priceMin;
            const priceMax = "do=" + request.priceMax;

            const olxUrl = {
                url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`,
                email: request.email,
                uuid: request.uniqueId
            }
            console.log(olxUrl.url);
            urls.push(olxUrl);
        }

        return urls;
    }
};


class Indexer {
    
/**
 * 
 * @param {String|Array} olxUrl single or array of URLS 
 * @param {String} email 
 * @param {Strung} uuid 
 */

    constructor(olxUrl, email, uuid) {
        this.olxUrl = olxUrl
        this.email = email
        this.uuid = uuid
    }

    async indexWithPagination(pageNumber = 1){

        let definitiveResults = [];
         console.log("This is olxUrl:" + this.olxUrl.url);
         const pageNr = this.olxUrl.url.match(/\d+$/);
        const indexers = this.iteratePages(pageNumber ? [pageNumber] : pageNr);
        console.log("This is page number:" + pageNr);
        // if (pageNr && Number(pageNr[0]) >= 20){
        //     console.log("Returning page numbermax");
        //     return;
        // }

        // if (pageNumber && pageNumber >= 30){
        //     console.log("Returning page numbermax");
        //     return;
        // }

       try{
        var lastPageNumber = indexers.lastPageNumber;

            return Promise.map(indexers.indexers, function (indexer) {
                console.log("Page number before async");
                console.log(pageNumber);
                return indexer.indexPage(pageNumber);
            }).then(async (results) => {
                let hasResults = false;
                results.forEach(result => {
                    if (!hasResults){
                        console.log("No results detected")
                        hasResults = result.hasResults 
                    }
                });
                

                // console.log(results);
                console.log("PageNumber");
                console.log(pageNumber);
                if (!hasResults){
                    console.log("HAS NO RESULTS");
                    // console.log(results);
                    return results             
                } else {
                    console.log("HAS RESULTS");
                    console.log(" Results PageNumber");
                    console.log(results[0].pageNumber)
                    const newResults = await this.indexWithPagination(results[0].pageNumber + 10);
                    // console.log(newResults);
                    Array.prototype.push.apply(results,newResults); 
                    return results;
                    // console.log("Idnexing seccond");
                    // // this.olxUrl.url = this.olxUrl.url.replace(/\d+$/, "")+lastPageNumber
                    // // // return await this.indexWithPagination();
                    // // Array.prototype.push.apply(definitiveResults,results); 
                    // definitiveResults = results;
                    // console.log("This is FIRST RESULT array");
                    // // console.log(results);
                    // console.log("This is definitive array after first results");
                    // console.log(definitiveResults);
                    // const pageNr2 = [10]
                    // const indexers2 = this.iteratePages([pageNr2[0]]);
                    // console.log("Indexers 2" + indexers2);
                    // console.log( indexers2);
                    // // const moreIndexers = this.iteratePages()
                    // return Promise.map(indexers2.indexers, function (seccondIndexer) {
                    //     console.log("Indexing seccond indexPage()");
                    //     return seccondIndexer.indexPage();
                    // }).then(async (results) => {
                    //     console.log("Indexing seccond results ");
                    //     Array.prototype.push.apply(definitiveResults,results); 
                    //     console.log("This is SECCOND RESULT array");
                    //     // console.log(results);
                    //     console.log("This is definitive array after SECCOND results");
                    //     console.log(definitiveResults);
                    //     return definitiveResults;
                    // });
                } 
            }); 
        } catch(e) {
            console.error("Error has accured", e);
        }

    }

    iteratePages(pageNr) {
        console.log("Entering iterate pages: page nr - " + pageNr );
        const indexers = [];
        let lastPageNumber;
        if (pageNr){
            for (let index = Number(pageNr[0]); index <= Number(pageNr[0]) + 10; index++) {
                lastPageNumber = index;

                // console.log("width number")
                // console.log("page number: " + Number(pageNr[0]))
                // console.log("index: " + index )
                console.log(this.olxUrl.url.replace(/\d+$/, "")+index)
                indexers.push(new Indexer({url : this.olxUrl.url.replace(/\d+$/, "")+index}))
                
            }
        } else {
            for (let index = 1; index <= 10; index++) {
                lastPageNumber = index;
                // console.log("widouth number")
                // console.log("index: " + index )
                // console.log(this.olxUrl.url+index)
                indexers.push(new Indexer({url :this.olxUrl.url+index}))
            }
        }
        return {indexers : indexers, lastPageNumber : lastPageNumber};
    }

    async indexPage(pageNumber) {
        console.log("Page number in index page")
        console.log(pageNumber);

        try {
            //TODO fix paging
            // console.log('Starting to index page: ' + pageNr);
            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
            console.log("Indexing page: " + this.olxUrl.url);
            const res = await fetch(this.olxUrl.url);
            const body = await res.text();
            const $ = cheerio.load(body);
            const hrefs = [];
            const results = [];
            let hasResults = false

            $('#rezultatipretrage').find('.listitem').each((i, elem) => {
                hasResults = true
                const href = $(elem).find('a').first().attr('href');
                hrefs.push(href);
            });

            // let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;

            // for (let i = 0; i < hrefs.length; i++) {
            //     console.log(`indexing: ${hrefs[i]}`);

            //     const singleData = await this.indexSingle(hrefs[i], this.olxUrl.email, this.olxUrl.uuid);

            //     if (singleData) {
            //         results.push(singleData);
            //     }
            // }

            // await this.sleep(this.olxUrl);
            // console.log('Finished indexing PAGE');
            // const singleIndex = [new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl)]
            // return Promise.map(singleIndex, function (indexer) {
            //     return indexer.indexSingle();
            // }).then(async (results) => {
               
            //     return results
            // })  
            console.log("this is hrefs for olxUrl" + this.olxUrl.url);

            console.log("NUMBER OF HREFS " + hrefs.length);
            // console.log("HREFS " + hrefs);
            console.log("HAS NO MORE RESULTS " + hasResults);

            return {hrefs : hrefs, hasResults : hasResults, pageNumber : pageNumber}
          
            // return results;
        } catch (e) {
            console.error('Exception caught:' + e);
        }
    }

    async indexAllHrefs () {

    }

    async indexSingle() {
        // try {
        //     const res = await fetch(url);
        //     const body = await res.text();
        //     const $ = cheerio.load(body);

        //     //TODO figure out what to do with username
        //     const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();

        //     // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
        //     //     return null;
        //     // }

        //     //TODO remove properties that are not needed, and add some if they are missing
        //     const title = $('#naslovartikla').text();
        //     const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();

        //     const price = $('#pc > p:nth-child(2)').text();
        //     const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
        //     const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
        //     const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
        //     const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
        //     const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');

        //     const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
        //     const time = $('time').attr('datetime');
        //     const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();

        //     const descriptions = $('.artikal_detaljniopis_tekst');
        //     // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
        //     const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
        //     const imgRe = /href":("[^"]*")/g;
        //     const matches = latLngRe.exec(body);
        //     let lng = '',
        //         lat = '';


        //     const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
        //     const parsePrice = (price) => parseFloat(price.replace(".", ""))


        //     // TODO we dont save images ??

        //     // const images = [];
        //     // const imgMatches = body.match(imgRe);

        //     // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
        //     //     let img = imgMatches[i].replace("href\":", "")
        //     //     img = img.replace("\"", "");
        //     //     img = img.replace("\"", "");
        //     //     images.push(img);
        //     // }

        //     // const uploadPromises = images.map(img => {
        //     //     const imgFixed = eval(`'${img}'`);
        //     //     return cloudinary.uploader.upload(eval(`'${img}'`));
        //     // });

        //     // const uploadResults = await Promise.all(uploadPromises);
        //     // const cloudinaryImages = uploadResults.map(ur => ur.url);

        //     if (matches && matches.length >= 3) {
        //         lat = matches[1];
        //         lng = matches[2];
        //     }

        //     const parsedPrice = parsePrice(price);

        //     const locationArray = location.split(",");
        //     const region = locationArray[0];
        //     const municipality = locationArray[1];

        //     const data = {
        //         realEstateType: this.getCategoryId(realEstateType),
        //         email : email,
        //         olxId: olxId,
        //         // category: category,
        //         url,
        //         title,
        //         price: isNaN(parsedPrice) ? price : parsedPrice,
        //         size: parseFloat(size),
        //         gardenSize: parseFloat(gardenSize),
        //         address,
        //         region,
        //         municipality,
        //         // adType: AD_TYPE_SALE,
        //         time,
        //         shortDescription: descriptions.first().text(),
        //         longDescription: descriptions.last().text(),
        //         lat,
        //         lng,
        //         loc: [parseFloat(lat), parseFloat(lng)],
        //         // images: cloudinaryImages
        //     };

        //     return data;
        // } catch (e) {
        //     console.error('Exception caught: ' + e.message);
        // }

        // return null;
        await this.sleep(this.olxUrl);
        console.log("Finished indexing single page");
        return {};
    }

    async sleep(ms) {
        console.log(ms);
        return new Promise(resolve => setTimeout(resolve, ms));
      }
}