Files
old-kivi/crawler/index.js

77 lines
2.1 KiB
JavaScript
Raw Normal View History

2016-11-07 11:17:48 +01:00
'use strict'
let fetch = require('node-fetch')
let jsonfile = require('jsonfile')
let cheerio = require('cheerio')
let fs = require('fs')
let twilio = require('twilio')
const accountSid = process.env.TWILLIO_ACCOUNT_SID;
const authToken = process.env.TWILLIO_AUTH_TOKEN;
if (!accountSid || !authToken) {
console.error('TWILLIO ACCOUNT ID or TOKEN missing!')
process.exit(-1);
}
const client = new twilio.RestClient(accountSid, authToken)
const kvadrataMax = 45
const url = `http://www.olx.ba/pretraga?id=23&kategorija=23&stranica=1&stanje=0&vrstapregleda=tabela&sort_order=desc&sort_po=datum&kanton=9&grad%5B%5D=3812&grad%5B%5D=5896&vrsta=samoprodaja&kvadrata_max=${kvadrataMax}`;
let data = {}
const receivers = ['+38761909238', '+38761818304'];
const sendSMSmessages = (url, price) => {
receivers.forEach(receiver => {
client.messages.create({
body: `Na piku je izasao interesantan stan: ${url} - Cijena: ${price}`,
to: receiver,
from: '+12034335458'
}, (err, message) => {
if (!err) {
console.log("Message sent to: ", receiver, message.sid);
} else {
console.log("Error while sending SMS to: ", receiver, err);
}
})
})
}
const start = () => {
data = jsonfile.readFileSync('stanovi.json')
console.log("Stanovi data successfully read: ", Object.keys(data).length, " objects")
setInterval(async () => {
console.log("doing");
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body)
let shouldSave = false
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
const id = $(elem).find('a').first().attr('href')
if (!data[id]) {
const cijena = $(elem).find(".cijena > .datum > span").first().text()
console.log('First time seen: ', id)
data[id] = {
seen: new Date(),
price: cijena
}
shouldSave = true
sendSMSmessages(id, cijena);
}
})
if (shouldSave) {
console.log("Syncing to the file...")
jsonfile.writeFileSync('stanovi.json', data)
}
}, 60000);
}
start()