diff --git a/crawler/pik.js b/crawler/build/olx.js similarity index 100% rename from crawler/pik.js rename to crawler/build/olx.js diff --git a/crawler/run.js b/crawler/run.js deleted file mode 100644 index 1a614a5..0000000 --- a/crawler/run.js +++ /dev/null @@ -1,76 +0,0 @@ -'use strict'; - -function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; } - -let fetch = require('node-fetch'); -let jsonfile = require('jsonfile'); -let cheerio = require('cheerio'); -let fs = require('fs'); -let twilio = require('twilio'); - -const accountSid = process.env.TWILLIO_ACCOUNT_SID; //'ACb21ff9b09a13a94ff0fb8d09ff7c950a' -const authToken = process.env.TWILLIO_AUTH_TOKEN; //'3fb7104aeb7faeb83520c969d7fa9578' - -if (!accountSid || !authToken) { - console.error('TWILLIO ACCOUNT ID or TOKEN missing!'); - process.exit(-1); -} -const client = new twilio.RestClient(accountSid, authToken); - -const kvadrataMax = 45; -const url = `http://www.olx.ba/pretraga?id=23&kategorija=23&stranica=1&stanje=0&vrstapregleda=tabela&sort_order=desc&sort_po=datum&kanton=9&grad%5B%5D=3812&grad%5B%5D=5896&vrsta=samoprodaja&kvadrata_max=${ kvadrataMax }`; - -let data = {}; - -const receivers = ['+38761909238', '+38761818304']; - -const sendSMSmessages = (url, price) => { - receivers.forEach(receiver => { - client.messages.create({ - body: `Na piku je izasao interesantan stan: ${ url } - Cijena: ${ price }`, - to: receiver, - from: '+12034335458' - }, (err, message) => { - if (!err) { - console.log("Message sent to: ", receiver, message.sid); - } else { - console.log("Error while sending SMS to: ", receiver, err); - } - }); - }); -}; - -const start = () => { - data = jsonfile.readFileSync('stanovi.json'); - console.log("Stanovi data successfully read: ", Object.keys(data).length, " objects"); - setInterval(_asyncToGenerator(function* () { - - console.log("doing"); - const res = yield fetch(url); - const body = yield res.text(); - - const $ = cheerio.load(body); - let shouldSave = false; - $('#rezultatipretrage').find('.listitem').each(function (i, elem) { - const id = $(elem).find('a').first().attr('href'); - if (!data[id]) { - const cijena = $(elem).find(".cijena > .datum > span").first().text(); - console.log('First time seen: ', id); - data[id] = { - seen: new Date(), - price: cijena - }; - shouldSave = true; - - sendSMSmessages(id, cijena); - } - }); - - if (shouldSave) { - console.log("Syncing to the file..."); - jsonfile.writeFileSync('stanovi.json', data); - } - }), 60000); -}; -start(); - diff --git a/crawler/stanovi.json b/crawler/stanovi.json deleted file mode 100644 index 7f7aa1d..0000000 --- a/crawler/stanovi.json +++ /dev/null @@ -1 +0,0 @@ -{"http://www.olx.ba/artikal/23212100/prostor-prodaje-jednoiposoban-stan-hrasno/":{"seen":"2016-10-18T12:56:09.329Z","price":"99.900 KM"},"http://www.olx.ba/artikal/23174319/prostor-prodaje-adaptiran-jednoiposoban-stan-hrasno/":{"seen":"2016-10-18T12:56:09.330Z","price":"94.000 KM"},"http://www.olx.ba/artikal/22450173/prostor-prodaje-garsonjera-na-drugom-spratu-mejtas/":{"seen":"2016-10-18T12:56:09.331Z","price":"73.000 KM"},"http://www.olx.ba/artikal/22330493/prostor-prodaje-namjesten-jednosoban-stan-sip-centar/":{"seen":"2016-10-18T12:56:09.332Z","price":"81.000 KM"},"http://www.olx.ba/artikal/23174998/stan-povrsine-40-m2-grbavica/":{"seen":"2016-10-18T12:56:09.333Z","price":"78.500 KM"},"http://www.olx.ba/artikal/23105950/stan-40-kvadrata-sarajevo-mojmilo-ul-olimpijska/":{"seen":"2016-10-18T12:56:09.333Z","price":"60.000 KM"},"http://www.olx.ba/artikal/13807104/stan-odobasina-41m2/":{"seen":"2016-10-18T12:56:09.334Z","price":"83.000 KM"},"http://www.olx.ba/artikal/22638577/jednosoban-stan-lozionicka-socijalno/":{"seen":"2016-10-18T12:56:09.335Z","price":"60.000 KM"},"http://www.olx.ba/artikal/20650394/ideea-nekretnine-jednosoban-stan-kosevsko-brdo/":{"seen":"2016-10-18T12:56:09.338Z","price":"80.000 KM"},"http://www.olx.ba/artikal/23251285/jednosoban-stan-na-grbavici/":{"seen":"2016-10-18T12:56:09.338Z","price":"86.000 KM"},"http://www.olx.ba/artikal/14852104/stan-42m2-na-jezeru-patriotske-lige/":{"seen":"2016-10-18T12:56:09.338Z","price":"85.000 KM"},"http://www.olx.ba/artikal/14475772/lijep-jednoiposoban-stan-sa-bastom-breka/":{"seen":"2016-10-18T12:56:09.339Z","price":"PO DOGOVORU"},"http://www.olx.ba/artikal/16845627/stan-u-blizini-medicinskog-fakulteta-centar/":{"seen":"2016-10-18T12:56:09.340Z","price":"70.000 KM"},"http://www.olx.ba/artikal/19840194/na-prodaju-jednosobni-stan-sa-bastom/":{"seen":"2016-10-18T12:56:09.340Z","price":"PO DOGOVORU"},"http://www.olx.ba/artikal/23070934/my-space-stan-kovacici-grbavicka-45-m2/":{"seen":"2016-10-18T12:56:09.341Z","price":"98.000 KM"},"http://www.olx.ba/artikal/18368584/stan-pionirska-dolina-nahorevska-novogradnja-33m2/":{"seen":"2016-10-18T12:56:09.342Z","price":"72.000 KM"},"http://www.olx.ba/artikal/18052751/my-space-stan-kosevsko-brdo-antuna-hangije-30-m2/":{"seen":"2016-10-18T12:56:09.342Z","price":"50.000 KM"},"http://www.olx.ba/artikal/19945927/my-space-stan-centar-branilaca-sarajeva-27-m2/":{"seen":"2016-10-18T12:56:09.343Z","price":"69.500 KM"},"http://www.olx.ba/artikal/18558052/my-space-stan-otoka-gradacacka-38m2/":{"seen":"2016-10-18T12:56:09.344Z","price":"79.000 KM"},"http://www.olx.ba/artikal/22469560/my-space-kosevo-bolnicka-42m2/":{"seen":"2016-10-18T12:56:09.345Z","price":"96.500 KM"},"http://www.olx.ba/artikal/18368528/my-space-kosevsko-brdo-jukiceva-30-m2/":{"seen":"2016-10-18T12:56:09.346Z","price":"59.000 KM"},"http://www.olx.ba/artikal/21679760/my-space-stan-centar-kosevsko-brdo-24m2-namjesten/":{"seen":"2016-10-18T12:56:09.346Z","price":"60.000 KM"},"http://www.olx.ba/artikal/18695823/my-space-stan-pofalici-humska-42-m2/":{"seen":"2016-10-18T12:56:09.347Z","price":"PO DOGOVORU"},"http://www.olx.ba/artikal/16120380/my-space-stan-kanc-prostor-breka-s-pasalica-40m2/":{"seen":"2016-10-18T12:56:09.347Z","price":"69.500 KM"},"http://www.olx.ba/artikal/16776051/my-space-stan-pofalici-humska-31-m2/":{"seen":"2016-10-18T12:56:09.347Z","price":"54.000 KM"},"http://www.olx.ba/artikal/21893611/my-space-stan-cengic-vila-dzemala-bijedica-34m2/":{"seen":"2016-10-18T12:56:09.348Z","price":"63.000 KM"},"http://www.olx.ba/artikal/22548370/my-space-stan-marijin-dvor-avde-jabucice-45-m2/":{"seen":"2016-10-18T12:56:09.349Z","price":"90.000 KM"},"http://www.olx.ba/artikal/21951224/my-space-stan-mejtas-armaganusa-45m2/":{"seen":"2016-10-18T12:56:09.349Z","price":"80.000 KM"},"http://www.olx.ba/artikal/18351145/my-space-stan-grbavica-grbavicka-30-43m2/":{"seen":"2016-10-18T12:56:09.350Z","price":"98.000 KM"},"http://www.olx.ba/artikal/16723372/my-space-stan-dolac-malta-marka-marulica-42m2/":{"seen":"2016-10-18T12:56:09.350Z","price":"83.000 KM"},"http://www.olx.ba/artikal/23184566/my-space-stan-novo-sarajevo-dzavida-haverica-29-m2/":{"seen":"2016-10-18T12:58:09.385Z","price":"53.500 KM"},"http://www.olx.ba/artikal/21325551/prodaje-se-jednoiposoban-stan-u-hrasnom/":{"seen":"2016-10-18T13:59:09.756Z","price":"90.000 KM"},"http://www.olx.ba/artikal/20007332/prodaje-se-jednoiposoban-stan-visnjik/":{"seen":"2016-10-18T13:59:09.756Z","price":"110.000 KM"},"http://www.olx.ba/artikal/21325736/prodaje-se-jednoiposoban-stan-na-kosevskom-brdu/":{"seen":"2016-10-18T14:00:09.936Z","price":"80.000 KM"},"http://www.olx.ba/artikal/21723510/prostor-prodaje-jednosoban-stan-kosevsko-brdo/":{"seen":"2016-10-18T14:33:52.427Z","price":"63.000 KM"},"http://www.olx.ba/artikal/22636983/prostor-prodaje-garsonjera-u-centru/":{"seen":"2016-10-18T14:34:53.406Z","price":"69.000 KM"},"http://www.olx.ba/artikal/23165580/home-invest-garsonjera-novo-sarajevo-prodaja/":{"seen":"2016-10-18T15:53:52.103Z","price":"43.000 KM"},"http://www.olx.ba/artikal/23164569/home-invest-garsonjera-sarajevo-centar-prodaja/":{"seen":"2016-10-18T15:53:52.112Z","price":"96.000 KM"},"http://www.olx.ba/artikal/22882860/stan-hrasno-porodice-ribar-38m2-78-000km/":{"seen":"2016-10-18T16:38:25.720Z","price":"78.000 KM"},"http://www.olx.ba/artikal/23263341/luxuri-partneri-prodaje-stan-u-centru/":{"seen":"2016-10-18T18:52:18.156Z","price":"139.000 KM"},"http://www.olx.ba/artikal/23264558/jednoiposoban-stan-hrasno/":{"seen":"2016-10-18T20:02:17.374Z","price":"79.500 KM"},"http://www.olx.ba/artikal/17574030/trosoban-stan-na-grbavici-81-m2/":{"seen":"2016-10-19T09:50:22.668Z","price":"238.000 KM"},"http://www.olx.ba/artikal/22713257/my-space-hrasno-porodice-ribar-38-m2/":{"seen":"2016-10-19T09:50:22.675Z","price":"78.000 KM"},"http://www.olx.ba/artikal/20693294/my-space-cengic-vila-dzemala-bijedica-38-m2/":{"seen":"2016-10-19T09:50:22.677Z","price":"63.000 KM"},"http://www.olx.ba/artikal/16087066/jednosoban-stan-38m2-hrasno-malta-grbavica-sarajevo/":{"seen":"2016-10-19T09:50:22.679Z","price":"75.000 KM"},"http://www.olx.ba/artikal/21699458/prodaje-se-stan-u-velesicima/":{"seen":"2016-10-19T09:50:22.680Z","price":"PO DOGOVORU"},"http://www.olx.ba/artikal/22723846/my-space-stan-grbavica-behdzeta-mutevelica-40-m2/":{"seen":"2016-10-19T13:22:23.417Z","price":"84.000 KM"},"http://www.olx.ba/artikal/20321110/my-space-kosevsko-brdo-renoviran-namjesten-36-m2/":{"seen":"2016-10-19T13:29:23.429Z","price":"72.000 KM"},"http://www.olx.ba/artikal/23123330/kovacici-jednosoban-stan-45-m2/":{"seen":"2016-10-19T14:49:23.685Z","price":"98.000 KM"},"http://www.olx.ba/artikal/22551147/stan-centar-alipasina-32-m2-odlicna-lokacija/":{"seen":"2016-10-19T17:17:24.188Z","price":"82.500 KM"},"http://www.olx.ba/artikal/23267816/izdajem-jednosoban-stan-40-kvadrata/":{"seen":"2016-10-19T18:58:13.062Z","price":"PO DOGOVORU"},"http://www.olx.ba/artikal/22994884/best-stan-38-m2-kosevsko-brdo-sarajevo/":{"seen":"2016-10-20T10:36:09.968Z","price":"85.000 KM"},"http://www.olx.ba/artikal/17251515/potraznja-stanova/":{"seen":"2016-10-20T10:36:09.977Z","price":"PO DOGOVORU"},"http://www.olx.ba/artikal/19313147/stan-u-novogradnji-na-mejtasu/":{"seen":"2016-10-20T10:46:09.581Z","price":"120.000 KM"},"http://www.olx.ba/artikal/19310241/stan-na-marijin-dvoru/":{"seen":"2016-10-20T10:46:09.583Z","price":"95.000 KM"},"http://www.olx.ba/artikal/21699051/stan-dvosoban-alipasina-sarajevo/":{"seen":"2016-10-20T10:46:09.585Z","price":"100.000 KM"},"http://www.olx.ba/artikal/22776245/prostor-prodaje-trosoban-stan-centar/":{"seen":"2016-10-20T11:38:42.920Z","price":"123.000 KM"},"http://www.olx.ba/artikal/22250543/adaptiran-jednoiposoban-stan-32-m2-vraca/":{"seen":"2016-10-21T10:53:09.384Z","price":"53.000 KM"},"http://www.olx.ba/artikal/22169788/stan-marijin-dvor-centar-vrazova-40-m2/":{"seen":"2016-10-21T10:53:09.391Z","price":"104.000 KM"},"http://www.olx.ba/artikal/22327728/dvosoban-stan-centar-vrtovi-sunca-45-m2/":{"seen":"2016-10-21T10:53:09.393Z","price":"86.089 KM"},"http://www.olx.ba/artikal/22197430/jednoiposoban-stan-k-brdo/":{"seen":"2016-10-21T10:53:09.395Z","price":"69.000 KM"},"http://www.olx.ba/artikal/20874732/40m2-u-pofalicima/":{"seen":"2016-10-21T10:53:09.396Z","price":"68.000 KM"},"http://www.olx.ba/artikal/22996121/jednosoban-stan-na-kosevskom-brdu/":{"seen":"2016-10-21T10:53:09.398Z","price":"65.000 KM"},"http://www.olx.ba/artikal/23122275/my-space-stan-hrasnica-33-m2/":{"seen":"2016-10-21T10:53:09.400Z","price":"47.000 KM"},"http://www.olx.ba/artikal/23290148/prodajem-jednoiposoban-stan-u-naselju-hrasno-sarajevo/":{"seen":"2016-10-21T10:53:09.401Z","price":"77.000 KM"},"http://www.olx.ba/artikal/23051249/jednosoban-stan-37m2-kosevsko-brdo-centar/":{"seen":"2016-10-21T10:53:09.405Z","price":"49.500 KM"},"http://www.olx.ba/artikal/18675829/dvosoban-stan-44m2-novogradnja-mejtas-centar/":{"seen":"2016-10-21T10:53:09.406Z","price":"110.000 KM"},"http://www.olx.ba/artikal/15717663/jednoiposoban-stan-45m2-vraca-novo-sarajevo/":{"seen":"2016-10-21T10:53:09.406Z","price":"73.000 KM"},"http://www.olx.ba/artikal/23046493/kosevsko-brdo-homelink-nekretnine-www-homelink-ba/":{"seen":"2016-10-21T10:53:09.407Z","price":"68.000 KM"},"http://www.olx.ba/artikal/23292413/prostor-prodaje-adaptiran-jednosoban-stan-grbavica/":{"seen":"2016-10-21T12:23:09.971Z","price":"99.000 KM"},"http://www.olx.ba/artikal/23272745/dvosoban-stan-pofalici-namjesten-50-metara-od-vilsonova/":{"seen":"2016-10-21T12:29:10.499Z","price":"350 KM"},"http://www.olx.ba/artikal/19177796/prodajem-jednosoban-stan-pofalici/":{"seen":"2016-10-21T16:11:12.306Z","price":"50.000 KM"},"http://www.olx.ba/artikal/15052611/prodajem-jednoiposoban-stan-u-ulici-porodice-ribara/":{"seen":"2016-10-21T22:36:40.320Z","price":"100.000 KM"},"http://www.olx.ba/artikal/21878487/prostor-prodaje-dvosoban-stan-centar/":{"seen":"2016-10-24T11:14:09.903Z","price":"117.000 KM"},"http://www.olx.ba/artikal/17700376/luxuripartneri-prodaje-stan-na-kosevu/":{"seen":"2016-10-24T11:14:09.911Z","price":"109.990 KM 98.990 KM"},"http://www.olx.ba/artikal/23300684/stan-grbavica/":{"seen":"2016-10-24T11:14:09.913Z","price":"89.000 KM"},"http://www.olx.ba/artikal/23216514/my-space-stan-velesici-mustafe-behmana-34-m2/":{"seen":"2016-10-24T11:14:09.914Z","price":"64.000 KM"},"http://www.olx.ba/artikal/18444705/my-space-grbavica-behdzeta-mutevelica-40-m2/":{"seen":"2016-10-24T11:14:09.915Z","price":"82.500 KM"},"http://www.olx.ba/artikal/18677836/stan-aerodromsko-naselje-39-m2-namjesten/":{"seen":"2016-10-24T11:14:09.917Z","price":"72.000 KM"},"http://www.olx.ba/artikal/21530592/my-space-stan-grbavicka-40-m2/":{"seen":"2016-10-24T11:14:09.918Z","price":"84.000 KM"},"http://www.olx.ba/artikal/21323647/my-space-grbavica-grbavicka-36-m2/":{"seen":"2016-10-24T11:14:09.922Z","price":"89.900 KM"},"http://www.olx.ba/artikal/20306911/garsonjera-za-izdavanje-5-min-od-cobanije/":{"seen":"2016-10-24T11:14:09.930Z","price":"200 KM"},"http://www.olx.ba/artikal/23169504/jednosoban-stan-kosevsko-brdo/":{"seen":"2016-10-24T11:14:09.931Z","price":"73.000 KM"},"http://www.olx.ba/artikal/22682764/stan-na-vracama/":{"seen":"2016-10-24T11:14:09.932Z","price":"49.000 KM"},"http://www.olx.ba/artikal/21500638/prodajem-jednosoban-stan-u-hrasnom/":{"seen":"2016-10-24T11:14:09.933Z","price":"87.000 KM"},"http://www.olx.ba/artikal/23324431/my-space-stan-muhameda-ef-pandze-37-m2/":{"seen":"2016-10-24T12:10:11.818Z","price":"53.000 KM"},"http://www.olx.ba/artikal/23328998/stan-dolac-malta-prvi-sprat/":{"seen":"2016-10-24T19:05:12.119Z","price":"81.000 KM"},"http://www.olx.ba/artikal/20882850/jednosoban-stan-33-kvadrata/":{"seen":"2016-10-24T21:58:45.368Z","price":"40.000 KM"},"http://www.olx.ba/artikal/21523079/cengic-vila-prodaje-se-stan-od-34-m2/":{"seen":"2016-10-25T13:16:15.911Z","price":"66.000 KM"},"http://www.olx.ba/artikal/22229983/jednosoban-stan-sa-lodjom-kosevsko-brdo/":{"seen":"2016-10-25T13:42:16.388Z","price":"49.000 KM"},"http://www.olx.ba/artikal/23335788/cengic-vila-stan-39m2/":{"seen":"2016-10-25T15:05:16.210Z","price":"84.000 KM"},"http://www.olx.ba/artikal/23433177/prostor-prodaje-dvosoban-stan-novogradnja-skenderija/":{"seen":"2016-11-03T14:59:26.024Z","price":"97.633 KM"},"http://www.olx.ba/artikal/23346262/my-space-stan-novo-sarajevo-vraca-27-m2/":{"seen":"2016-11-03T14:59:26.033Z","price":"52.000 KM"},"http://www.olx.ba/artikal/13244071/stan-34-m2-cengic-vila-novogradnja/":{"seen":"2016-11-03T14:59:26.035Z","price":"69.000 KM"},"http://www.olx.ba/artikal/18837363/sanesko-prodaje-jednosoban-stan-na-vracama/":{"seen":"2016-11-03T14:59:26.036Z","price":"53.000 KM"},"http://www.olx.ba/artikal/21565032/sanesko-prodaje-jednoiposoban-stan-na-grbavici/":{"seen":"2016-11-03T14:59:26.037Z","price":"79.000 KM"},"http://www.olx.ba/artikal/16686870/stan-breka-sarajevo/":{"seen":"2016-11-03T14:59:26.039Z","price":"PO DOGOVORU"},"http://www.olx.ba/artikal/23352350/prodajem-namjesten-stan-u-centru-27-m2-plus-ostava-15-m2/":{"seen":"2016-11-03T14:59:26.043Z","price":"34.500 KM"},"http://www.olx.ba/artikal/23188478/stan-pofalici-42m2-prodaje-se/":{"seen":"2016-11-03T14:59:26.044Z","price":"85.000 KM"},"http://www.olx.ba/artikal/21147546/jednosoban-stan/":{"seen":"2016-11-03T14:59:26.046Z","price":"47.000 KM"},"http://www.olx.ba/artikal/23422969/jednosoban-stan/":{"seen":"2016-11-03T14:59:26.048Z","price":"75.000 KM"},"http://www.olx.ba/artikal/23420119/stan-novo-sarajevo-hrasno/":{"seen":"2016-11-03T14:59:26.051Z","price":"33.000 KM"},"http://www.olx.ba/artikal/19775897/jednoiposoban-stan-grbavica/":{"seen":"2016-11-03T14:59:26.055Z","price":"75.000 KM"},"http://www.olx.ba/artikal/12479738/prodajem-stan-na-marijin-dvoru/":{"seen":"2016-11-03T14:59:26.056Z","price":"98.000 KM"}}