senaduka/old-kivi
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

crawlers and savers

Browse Source
This commit is contained in:
Senad Uka
2016-11-10 13:04:45 +01:00
parent 7a16a0191c
commit 5a819cc743
14 changed files with 2695 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
node_modules node_modules
.DS_Store .DS_Store
crawler/build

4
README.md
View File

@@ -22,8 +22,8 @@ Ukljucen je webpack hot module replacement + webpack-dev-server tako da se sve i
Trenutno postoji samo jedan crawler a to je `olx.js` Trenutno postoji samo jedan crawler a to je `olx.js`
1. cd crawler 1. cd crawler
2. babel olx.js > build/olx.js 2. npm run dev
3. node build/olx.js 3. node build/crawler.js
4. profit! 4. profit!
Trenutno se rezultati crawlanja spasavaju u JSON file, naravno ovo moramo cim prije promjeniti. Trenutno se rezultati crawlanja spasavaju u JSON file, naravno ovo moramo cim prije promjeniti.

3
crawler/.babelrc
View File

@@ -1,3 +1,4 @@
{ {
"plugins": ["transform-async-to-generator"] "plugins": ["transform-async-to-generator"],
"presets":["es2015"]
} }

4
crawler/.env Normal file
View File

@@ -0,0 +1,4 @@
MONGO_URL='mongodb://localhost:27017/kivi'
OLX_FROM_PAGE=8
OLX_TO_PAGE=8
OLX_MAX_RESULTS=1000

48
crawler/crawl.js Normal file
View File

@@ -0,0 +1,48 @@
'use strict'
/*
Entry point for crawling functionality
All communication between crawlers and savers is here
All environment specific configuration is read here and
passed to the crawlers and savers.
*/
import {
install
} from 'source-map-support';
import 'dotenv/config';
import OlxCrawler from './specific/olx';
import MongoSaver from './savers/mongo'
install(); // for source maps to work
let crawlers = [
new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS)
];
let savers = [
new MongoSaver(process.env.MONGO_URL)
];
let done = 0;
for (let crawler of crawlers) {
try {
crawler.crawl().then((results) => {
for (let saver of savers) {
try {
saver.open();
saver.save(results);
} catch (e) {
console.log("Error saving. Trying next saver! ", e);
}
}
});
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
}
}
for (let saver of savers) {
saver.close();
}

76
crawler/index.js
View File

@@ -1,76 +0,0 @@
'use strict'
let fetch = require('node-fetch')
let jsonfile = require('jsonfile')
let cheerio = require('cheerio')
let fs = require('fs')
let twilio = require('twilio')
const accountSid = process.env.TWILLIO_ACCOUNT_SID;
const authToken = process.env.TWILLIO_AUTH_TOKEN;
if (!accountSid || !authToken) {
console.error('TWILLIO ACCOUNT ID or TOKEN missing!')
process.exit(-1);
}
const client = new twilio.RestClient(accountSid, authToken)
const kvadrataMax = 45
const url = `http://www.olx.ba/pretraga?id=23&kategorija=23&stranica=1&stanje=0&vrstapregleda=tabela&sort_order=desc&sort_po=datum&kanton=9&grad%5B%5D=3812&grad%5B%5D=5896&vrsta=samoprodaja&kvadrata_max=${kvadrataMax}`;
let data = {}
const receivers = ['+38761909238', '+38761818304'];
const sendSMSmessages = (url, price) => {
receivers.forEach(receiver => {
client.messages.create({
body: `Na piku je izasao interesantan stan: ${url} - Cijena: ${price}`,
to: receiver,
from: '+12034335458'
}, (err, message) => {
if (!err) {
console.log("Message sent to: ", receiver, message.sid);
} else {
console.log("Error while sending SMS to: ", receiver, err);
}
})
})
}
const start = () => {
data = jsonfile.readFileSync('stanovi.json')
console.log("Stanovi data successfully read: ", Object.keys(data).length, " objects")
setInterval(async () => {
console.log("doing");
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body)
let shouldSave = false
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
const id = $(elem).find('a').first().attr('href')
if (!data[id]) {
const cijena = $(elem).find(".cijena > .datum > span").first().text()
console.log('First time seen: ', id)
data[id] = {
seen: new Date(),
price: cijena
}
shouldSave = true
sendSMSmessages(id, cijena);
}
})
if (shouldSave) {
console.log("Syncing to the file...")
jsonfile.writeFileSync('stanovi.json', data)
}
}, 60000);
}
start()

124
crawler/olx.js
View File

@@ -1,124 +0,0 @@
'use strict'
let fetch = require('node-fetch');
let jsonfile = require('jsonfile');
let cheerio = require('cheerio');
let fs = require('fs');
const pagesToTake = 10;
async function indexSingle(url) {
try {
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
const title = $('#naslovartikla').text();
const price = $('#pc > p:nth-child(2)').text();
const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
const time = $('time').attr('datetime');
const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
const descriptions = $('.artikal_detaljniopis_tekst');
const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
const imgRe = /href":("[^"]*")/g;
const matches = latLngRe.exec(body);
let lng = '',
lat = '';
const images = [];
const imgMatches = body.match(imgRe);
console.log(imgMatches);
for(let i = 0; imgMatches && i < imgMatches.length; i++) {
let img = imgMatches[i].replace("href\":", "")
img = img.replace("\"", "");
img = img.replace("\"", "");
images.push(img);
}
if (matches && matches.length >= 3) {
lat = matches[1];
lng = matches[2];
}
const data = {
title,
price,
size,
rooms,
address,
location,
adType,
time,
olxId,
shortDescription: descriptions.first().text(),
longDescription: descriptions.last().text(),
lat,
lng,
images
};
return data;
} catch(e) {
console.error('Exception caught: ' + e);
}
return null;
}
async function indexPage(pageNr) {
try {
console.log('Starting to index page: ' + pageNr);
const url = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
const hrefs = [];
const results = {};
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
const href = $(elem).find('a').first().attr('href');
hrefs.push(href);
});
for(let i = 0; i < hrefs.length; i++) {
console.log(`indexing: ${hrefs[i]}`);
const singleData = await indexSingle(hrefs[i]);
if (singleData) {
results[hrefs[i]] = singleData;
}
await sleep(500);
}
jsonfile.writeFileSync(`izdavanje-sarajevo-page-${pageNr}.json`, results);
} catch(e) {
console.error('Exception caught:' + e);
}
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async function indexPages(start, end) {
for(let i = start; i <= end; i++) {
await indexPage(i);
sleep(5000);
}
}
indexPages(8, 10);
//indexSingle('http://www.olx.ba/artikal/23198642/trosoban-stan-centar-josipa-vancasa/');

48
crawler/package.json
View File

@@ -1,20 +1,32 @@
{ {
"name": "stan", "name": "stan",
"version": "1.0.0", "version": "1.0.0",
"description": "", "description": "",
"main": "index.js", "main": "index.js",
"dependencies": { "dependencies": {
"babel-plugin-transform-async-to-generator": "^6.16.0", "babel": "^6.5.2",
"cheerio": "^0.22.0", "babel-core": "^6.18.2",
"fetch": "^1.1.0", "babel-loader": "^6.2.7",
"jsonfile": "^2.4.0", "babel-plugin-transform-async-to-generator": "^6.16.0",
"node-fetch": "^1.6.3", "babel-polyfill": "^6.16.0",
"twilio": "^2.11.0" "babel-preset-es2015": "^6.18.0",
}, "cheerio": "^0.22.0",
"devDependencies": {}, "dotenv": "^2.0.0",
"scripts": { "fetch": "^1.1.0",
"test": "echo \"Error: no test specified\" && exit 1" "json-loader": "^0.5.4",
}, "mongodb": "^2.2.11",
"author": "", "node-fetch": "^1.6.3",
"license": "ISC" "source-map-support": "^0.4.6",
"twilio": "^2.11.0"
},
"devDependencies": {
"webpack": "^1.13.3"
},
"scripts": {
"dev": "webpack",
"prod": "webpack -p",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC"
} }

58
crawler/savers/mongo.js Normal file
View File

@@ -0,0 +1,58 @@
import MongoClient from 'mongodb';
export default class MongoSaver {
constructor(url) {
this.url = url;
}
connect() {
let saver = this;
return new Promise((resolve, reject) => {
if (!saver.ready) {
MongoClient.connect(saver.url, (err, db) => {
if (err) {
console.log('Unable to connect to the mongoDB server. Error:', err);
reject(err);
} else {
console.log('Connection established to', url);
saver.db = db;
saver.collection = db.collection('results');
saver.ready = true;
resolve();
}
});
} else {
resolve();
}
});
}
save(results) {
let resultsForMongo = Object.keys(results).map((key) => {
return results[key]
});
this.collection.insert(resultsForMongo);
}
async close() {
try {
//Close connection
await this.disconnect();
} catch (e) {
console.log("error closing", e);
}
}
async disconnect() {
return new Promise((resolve, reject) => {
this.db.close();
resolve();
});
}
async open() {
await this.connect();
}
}

110
crawler/build/olx.js → crawler/specific/olx.js
View File

@@ -1,10 +1,24 @@
'use strict'; 'use strict'
let indexSingle = (() => { let fetch = require('node-fetch');
var _ref = _asyncToGenerator(function* (url) { let cheerio = require('cheerio');
let fs = require('fs');
const pagesToTake = 10;
export default class OlxCrawler {
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
this.fromPage = fromPage;
this.toPage = toPage;
this.maxResults = maxResults;
}
async indexSingle(url) {
try { try {
const res = yield fetch(url); const res = await fetch(url);
const body = yield res.text(); const body = await res.text();
const $ = cheerio.load(body); const $ = cheerio.load(body);
const title = $('#naslovartikla').text(); const title = $('#naslovartikla').text();
@@ -21,17 +35,15 @@ let indexSingle = (() => {
const descriptions = $('.artikal_detaljniopis_tekst'); const descriptions = $('.artikal_detaljniopis_tekst');
const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g; const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
const imgRe = /href":("[^"]*")/g; const imgRe = /href":("[^"]*")/g;
//href":("[^"]*")]")"
const matches = latLngRe.exec(body); const matches = latLngRe.exec(body);
let lng = '', let lng = '',
lat = ''; lat = '';
const images = []; const images = [];
const imgMatches = body.match(imgRe); //imgRe.exec(body); const imgMatches = body.match(imgRe);
console.log(imgMatches);
for (let i = 0; imgMatches && i < imgMatches.length; i++) { for (let i = 0; imgMatches && i < imgMatches.length; i++) {
let img = imgMatches[i].replace("href\":", ""); let img = imgMatches[i].replace("href\":", "")
img = img.replace("\"", ""); img = img.replace("\"", "");
img = img.replace("\"", ""); img = img.replace("\"", "");
images.push(img); images.push(img);
@@ -43,6 +55,7 @@ let indexSingle = (() => {
} }
const data = { const data = {
url,
title, title,
price, price,
size, size,
@@ -59,89 +72,68 @@ let indexSingle = (() => {
images images
}; };
console.log(data);
return data; return data;
} catch (e) { } catch (e) {
console.error('Exception caught: ' + e); console.error('Exception caught: ' + e);
} }
return null; return null;
}); }
return function indexSingle(_x) { async indexPage(pageNr, maxResults = 1000) {
return _ref.apply(this, arguments);
};
})();
let indexPage = (() => {
var _ref2 = _asyncToGenerator(function* (pageNr) {
try { try {
console.log('Starting to index page: ' + pageNr); console.log('Starting to index page: ' + pageNr);
const url = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${ pageNr }`; const url = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
const res = yield fetch(url); const res = await fetch(url);
const body = yield res.text(); const body = await res.text();
const $ = cheerio.load(body); const $ = cheerio.load(body);
const hrefs = []; const hrefs = [];
const results = {}; const results = {};
$('#rezultatipretrage').find('.listitem').each(function (i, elem) { $('#rezultatipretrage').find('.listitem').each((i, elem) => {
const href = $(elem).find('a').first().attr('href'); const href = $(elem).find('a').first().attr('href');
hrefs.push(href); hrefs.push(href);
}); });
console.log('number to index: ' + hrefs.length); let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
for (let i = 0; i < hrefs.length; i++) {
console.log(`indexing: ${ hrefs[i] }`);
const singleData = yield indexSingle(hrefs[i]); for (let i = 0; i < actualNoOfResults; i++) {
console.log(`indexing: ${hrefs[i]}`);
const singleData = await this.indexSingle(hrefs[i]);
if (singleData) { if (singleData) {
results[hrefs[i]] = singleData; results[hrefs[i]] = singleData;
} }
yield sleep(500); await this.sleep(500);
} }
jsonfile.writeFileSync(`izdavanje-sarajevo-page-${ pageNr }.json`, results); return results;
} catch (e) { } catch (e) {
console.error('Exception caught:' + e); console.error('Exception caught:' + e);
} }
}); }
return function indexPage(_x2) { async sleep(ms) {
return _ref2.apply(this, arguments); return new Promise(resolve => setTimeout(resolve, ms));
}; }
})();
let indexPages = (() => { async indexPages(start, end, maxResults = 1000) {
var _ref3 = _asyncToGenerator(function* (start, end) { let results = {};
for (let i = start; i <= end; i++) { for (let i = start; i <= end; i++) {
console.log('Start page: ', i); let result = await this.indexPage(i, numberOfResults);
yield indexPage(i); Object.assign(results, result)
console.log('Done with page!'); await this.sleep(5000);
sleep(5000);
} }
}); return results;
}
return function indexPages(_x3, _x4) { async crawl() {
return _ref3.apply(this, arguments); let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
}; return results;
})(); }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
let fetch = require('node-fetch');
let jsonfile = require('jsonfile');
let cheerio = require('cheerio');
let fs = require('fs');
const pagesToTake = 10;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
} }
indexPages(8, 10);
//indexSingle('http://www.olx.ba/artikal/23198642/trosoban-stan-centar-josipa-vancasa/'); //indexSingle('http://www.olx.ba/artikal/23198642/trosoban-stan-centar-josipa-vancasa/');

23
crawler/webpack.config.js Normal file
View File

@@ -0,0 +1,23 @@
module.exports = {
entry: ['babel-polyfill', './crawl.js'],
target: 'node',
output: {
path: __dirname + "/build",
filename: "crawler.js",
devtool: 'source-map'
},
module: {
loaders: [{
test: /.js?$/,
loader: 'babel-loader',
exclude: /node_modules/,
presets: ['es2015'],
plugins: ['transform-async-to-generator']
}, {
test: /.json?$/,
loader: 'json-loader',
}]
}
}

1528
crawler/yarn.lock
View File

File diff suppressed because it is too large Load Diff

27
npm-debug.log Normal file
View File

@@ -0,0 +1,27 @@
0 info it worked if it ends with ok
1 verbose cli [ '/home/senadu/.nvm/versions/node/v5.4.1/bin/node',
1 verbose cli '/home/senadu/.nvm/versions/node/v5.4.1/bin/npm',
1 verbose cli 'run',
1 verbose cli 'dev' ]
2 info using npm@3.3.12
3 info using node@v5.4.1
4 verbose config Skipping project config: /home/senadu/.npmrc. (matches userconfig)
5 verbose stack Error: missing script: dev
5 verbose stack at run (/home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/lib/run-script.js:147:19)
5 verbose stack at /home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/lib/run-script.js:57:5
5 verbose stack at /home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/node_modules/read-package-json/read-json.js:345:5
5 verbose stack at checkBinReferences_ (/home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/node_modules/read-package-json/read-json.js:309:45)
5 verbose stack at final (/home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/node_modules/read-package-json/read-json.js:343:3)
5 verbose stack at then (/home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/node_modules/read-package-json/read-json.js:113:5)
5 verbose stack at ReadFileContext.<anonymous> (/home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/node_modules/read-package-json/read-json.js:284:20)
5 verbose stack at ReadFileContext.callback (/home/senadu/.nvm/versions/node/v5.4.1/lib/node_modules/npm/node_modules/graceful-fs/graceful-fs.js:76:16)
5 verbose stack at FSReqWrap.readFileAfterOpen [as oncomplete] (fs.js:324:13)
6 verbose cwd /home/senadu/projects/saburly/kivi
7 error Linux 4.4.0-45-generic
8 error argv "/home/senadu/.nvm/versions/node/v5.4.1/bin/node" "/home/senadu/.nvm/versions/node/v5.4.1/bin/npm" "run" "dev"
9 error node v5.4.1
10 error npm v3.3.12
11 error missing script: dev
12 error If you need help, you may report this error at:
12 error <https://github.com/npm/npm/issues>
13 verbose exit [ 1, true ]

964
web/yarn.lock
View File

File diff suppressed because it is too large Load Diff