Crawler cleanup

crawler/run.js

@@ -1,76 +0,0 @@
-'use strict';
-
-function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
-
-let fetch = require('node-fetch');
-let jsonfile = require('jsonfile');
-let cheerio = require('cheerio');
-let fs = require('fs');
-let twilio = require('twilio');
-
-const accountSid = process.env.TWILLIO_ACCOUNT_SID; //'ACb21ff9b09a13a94ff0fb8d09ff7c950a'
-const authToken = process.env.TWILLIO_AUTH_TOKEN; //'3fb7104aeb7faeb83520c969d7fa9578'
-
-if (!accountSid || !authToken) {
-  console.error('TWILLIO ACCOUNT ID or TOKEN missing!');
-  process.exit(-1);
-}
-const client = new twilio.RestClient(accountSid, authToken);
-
-const kvadrataMax = 45;
-const url = `http://www.olx.ba/pretraga?id=23&kategorija=23&stranica=1&stanje=0&vrstapregleda=tabela&sort_order=desc&sort_po=datum&kanton=9&grad%5B%5D=3812&grad%5B%5D=5896&vrsta=samoprodaja&kvadrata_max=${ kvadrataMax }`;
-
-let data = {};
-
-const receivers = ['+38761909238', '+38761818304'];
-
-const sendSMSmessages = (url, price) => {
-  receivers.forEach(receiver => {
-    client.messages.create({
-      body: `Na piku je izasao interesantan stan: ${ url } - Cijena: ${ price }`,
-      to: receiver,
-      from: '+12034335458'
-    }, (err, message) => {
-      if (!err) {
-        console.log("Message sent to: ", receiver, message.sid);
-      } else {
-        console.log("Error while sending SMS to: ", receiver, err);
-      }
-    });
-  });
-};
-
-const start = () => {
-  data = jsonfile.readFileSync('stanovi.json');
-  console.log("Stanovi data successfully read: ", Object.keys(data).length, " objects");
-  setInterval(_asyncToGenerator(function* () {
-
-    console.log("doing");
-    const res = yield fetch(url);
-    const body = yield res.text();
-
-    const $ = cheerio.load(body);
-    let shouldSave = false;
-    $('#rezultatipretrage').find('.listitem').each(function (i, elem) {
-      const id = $(elem).find('a').first().attr('href');
-      if (!data[id]) {
-        const cijena = $(elem).find(".cijena > .datum > span").first().text();
-        console.log('First time seen: ', id);
-        data[id] = {
-          seen: new Date(),
-          price: cijena
-        };
-        shouldSave = true;
-
-        sendSMSmessages(id, cijena);
-      }
-    });
-
-    if (shouldSave) {
-      console.log("Syncing to the file...");
-      jsonfile.writeFileSync('stanovi.json', data);
-    }
-  }), 60000);
-};
-start();
-