Files
old-kivi/crawler/run.js
Edin Dazdarevic 8f613f1f73 Initial commit
2016-11-07 11:17:48 +01:00

77 lines
2.7 KiB
JavaScript

'use strict';
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
let fetch = require('node-fetch');
let jsonfile = require('jsonfile');
let cheerio = require('cheerio');
let fs = require('fs');
let twilio = require('twilio');
const accountSid = process.env.TWILLIO_ACCOUNT_SID; //'ACb21ff9b09a13a94ff0fb8d09ff7c950a'
const authToken = process.env.TWILLIO_AUTH_TOKEN; //'3fb7104aeb7faeb83520c969d7fa9578'
if (!accountSid || !authToken) {
console.error('TWILLIO ACCOUNT ID or TOKEN missing!');
process.exit(-1);
}
const client = new twilio.RestClient(accountSid, authToken);
const kvadrataMax = 45;
const url = `http://www.olx.ba/pretraga?id=23&kategorija=23&stranica=1&stanje=0&vrstapregleda=tabela&sort_order=desc&sort_po=datum&kanton=9&grad%5B%5D=3812&grad%5B%5D=5896&vrsta=samoprodaja&kvadrata_max=${ kvadrataMax }`;
let data = {};
const receivers = ['+38761909238', '+38761818304'];
const sendSMSmessages = (url, price) => {
receivers.forEach(receiver => {
client.messages.create({
body: `Na piku je izasao interesantan stan: ${ url } - Cijena: ${ price }`,
to: receiver,
from: '+12034335458'
}, (err, message) => {
if (!err) {
console.log("Message sent to: ", receiver, message.sid);
} else {
console.log("Error while sending SMS to: ", receiver, err);
}
});
});
};
const start = () => {
data = jsonfile.readFileSync('stanovi.json');
console.log("Stanovi data successfully read: ", Object.keys(data).length, " objects");
setInterval(_asyncToGenerator(function* () {
console.log("doing");
const res = yield fetch(url);
const body = yield res.text();
const $ = cheerio.load(body);
let shouldSave = false;
$('#rezultatipretrage').find('.listitem').each(function (i, elem) {
const id = $(elem).find('a').first().attr('href');
if (!data[id]) {
const cijena = $(elem).find(".cijena > .datum > span").first().text();
console.log('First time seen: ', id);
data[id] = {
seen: new Date(),
price: cijena
};
shouldSave = true;
sendSMSmessages(id, cijena);
}
});
if (shouldSave) {
console.log("Syncing to the file...");
jsonfile.writeFileSync('stanovi.json', data);
}
}), 60000);
};
start();