senaduka/old-kivi
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Random stuff

Browse Source
This commit is contained in:
Edin Dazdarevic
2017-04-12 15:33:30 +02:00
parent 6a4c02d01a
commit 6ef680a7d3
3 changed files with 9 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/crawl.js
View File

@@ -20,7 +20,7 @@ install(); // for source maps to work
let crawlers = [
//new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
new ProstorCrawler(process.env.PROSTOR_FROM_PAGE, process.env.PROSTOR_TO_PAGE, process.env.PROSTOR_MAX_RESULTS)
new ProstorCrawler(parseInt(process.env.PROSTOR_FROM_PAGE), parseInt(process.env.PROSTOR_TO_PAGE), parseInt(process.env.PROSTOR_MAX_RESULTS))
];
let savers = [

16
crawler/specific/prostor.js
View File

@@ -30,8 +30,6 @@ export default class ProstorCrawler {
const body = await res.text();
const $ = cheerio.load(body);
console.log('bailing...');
const title = $('#nav_center_sub > div.content_area_1_left > div:nth-child(1) > h1').text();
const category = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(3) > div:nth-child(4) > div.size_rs > span').text();
@@ -105,12 +103,12 @@ export default class ProstorCrawler {
//images.push(img);
//}
const uploadPromises = images.map(img => {
return cloudinary.uploader.upload(img);
});
//const uploadPromises = images.map(img => {
//return cloudinary.uploader.upload(img);
//});
const uploadResults = await Promise.all(uploadPromises);
const cloudinaryImages = uploadResults.map(ur => ur.url);
//const uploadResults = await Promise.all(uploadPromises);
//const cloudinaryImages = uploadResults.map(ur => ur.url);
const parsedPrice = parsePrice(price);
@@ -138,7 +136,8 @@ export default class ProstorCrawler {
lat,
lng,
loc: [parseFloat(lat), parseFloat(lng)],
images: cloudinaryImages
//images: cloudinaryImages
images
};
console.log(data);
@@ -231,4 +230,3 @@ export default class ProstorCrawler {
return results;
}
}