senaduka/old-kivi
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Use cloudinary

Browse Source
This commit is contained in:
Edin Dazdarevic
2017-04-10 02:50:40 +02:00
parent 5f4e3a01d3
commit 738720aa13
7 changed files with 72 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
crawler/package.json
View File

@@ -1,32 +1,33 @@
{
"name": "stan",
"version": "1.0.0",
"description": "",
"main": "index.js",
"dependencies": {
"babel": "^6.5.2",
"babel-core": "^6.18.2",
"babel-loader": "^6.2.7",
"babel-plugin-transform-async-to-generator": "^6.16.0",
"babel-polyfill": "^6.16.0",
"babel-preset-es2015": "^6.18.0",
"cheerio": "^0.22.0",
"dotenv": "^2.0.0",
"fetch": "^1.1.0",
"json-loader": "^0.5.4",
"mongodb": "^2.2.11",
"node-fetch": "^1.6.3",
"source-map-support": "^0.4.6",
"twilio": "^2.11.0"
},
"devDependencies": {
"webpack": "^1.13.3"
},
"scripts": {
"dev": "webpack",
"prod": "webpack -p",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC"
"name": "stan",
"version": "1.0.0",
"description": "",
"main": "index.js",
"dependencies": {
"babel": "^6.5.2",
"babel-core": "^6.18.2",
"babel-loader": "^6.2.7",
"babel-plugin-transform-async-to-generator": "^6.16.0",
"babel-polyfill": "^6.16.0",
"babel-preset-es2015": "^6.18.0",
"cheerio": "^0.22.0",
"cloudinary": "^1.8.0",
"dotenv": "^2.0.0",
"fetch": "^1.1.0",
"json-loader": "^0.5.4",
"mongodb": "^2.2.11",
"node-fetch": "^1.6.3",
"source-map-support": "^0.4.6",
"twilio": "^2.11.0"
},
"devDependencies": {
"webpack": "^1.13.3"
},
"scripts": {
"dev": "webpack",
"prod": "webpack -p",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC"
}

5
crawler/savers/mongo.js
View File

@@ -33,8 +33,9 @@ export default class MongoSaver {
return results[key]
});
this.collection.update({ url: results.url }, resultsForMongo, { upsert: true });
// this.collection.insert(resultsForMongo);
for(const doc of resultsForMongo) {
this.collection.update({ url: doc.url }, doc, { upsert: true });
}
}
async close() {

13
crawler/specific/olx.js
View File

@@ -3,6 +3,7 @@
let fetch = require('node-fetch');
let cheerio = require('cheerio');
let fs = require('fs');
let cloudinary = require('cloudinary');
import {
AD_TYPE_SALE,
@@ -68,6 +69,14 @@ export default class OlxCrawler {
images.push(img);
}
const uploadPromises = images.map(img => {
const imgFixed = eval(`'${img}'`);
return cloudinary.uploader.upload(eval(`'${img}'`));
});
const uploadResults = await Promise.all(uploadPromises);
const cloudinaryImages = uploadResults.map(ur => ur.url);
if (matches && matches.length >= 3) {
lat = matches[1];
lng = matches[2];
@@ -99,12 +108,12 @@ export default class OlxCrawler {
lat,
lng,
loc: [parseFloat(lat), parseFloat(lng)],
images
images: cloudinaryImages
};
return data;
} catch (e) {
console.error('Exception caught: ' + e);
console.error('Exception caught: ' + e.message);
}
return null;

15
crawler/yarn.lock
View File

@@ -745,6 +745,13 @@ clone@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.2.tgz#260b7a99ebb1edfe247538175f783243cb19d149"
cloudinary:
version "1.8.0"
resolved "https://registry.yarnpkg.com/cloudinary/-/cloudinary-1.8.0.tgz#52e7a18dfe3c0ece01abc71341be95d0ea198d85"
dependencies:
lodash "3.10.x"
q "1.4.x"
code-point-at@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
@@ -1532,6 +1539,10 @@ lodash@^4.14.0, lodash@^4.2.0:
version "4.16.6"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.6.tgz#d22c9ac660288f3843e16ba7d2b5d06cca27d777"
lodash@3.10.x:
version "3.10.1"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
longest@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097"
@@ -1860,6 +1871,10 @@ q@0.9.7:
version "0.9.7"
resolved "https://registry.yarnpkg.com/q/-/q-0.9.7.tgz#4de2e6cb3b29088c9e4cbc03bf9d42fb96ce2f75"
q@1.4.x:
version "1.4.1"
resolved "https://registry.yarnpkg.com/q/-/q-1.4.1.tgz#55705bcd93c5f3673530c2c2cbc0c2b3addc286e"
qs@~6.2.0:
version "6.2.1"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.2.1.tgz#ce03c5ff0935bc1d9d69a9f14cbd18e568d67625"

3
web/components/Gallery.js
View File

@@ -1,4 +1,5 @@
import React from 'react';
import {galleryImageUrl} from '../lib/helpers';
export default class Gallery extends React.Component {
onPrevClick (e) {
@@ -24,7 +25,7 @@ export default class Gallery extends React.Component {
return (
<div className="ld-image-container">
<img src={images[imageIndex]}></img>
<img src={galleryImageUrl(images[imageIndex])}></img>
{showPrev ?
<div
className='prev-button'

6
web/components/Listings.js
View File

@@ -1,6 +1,6 @@
import React from 'react';
import {findDOMNode} from 'react-dom';
import {formatPrice} from '../lib/helpers';
import {formatPrice, listingImageUrl} from '../lib/helpers';
import {loadListing} from '../lib/api';
export default class Listings extends React.Component {
@@ -22,8 +22,6 @@ export default class Listings extends React.Component {
}
onListingClick(id) {
loadListing(id).then(l => l.text()).then(l => {
this.props.dispatch({type: 'UPDATE_ROUTE', action: {
toDispatch: {
@@ -79,7 +77,7 @@ export default class Listings extends React.Component {
className="property-list-item"
onClick={this.onListingClick.bind(this, l._id)}>
<div className="pli-image">
<img src={images[0]} alt=""></img>
<img src={listingImageUrl(images[0])} alt=""></img>
</div>
<div className="pli-details">
<div className="price">{formatPrice(l.price)}</div>

8
web/lib/helpers.js
View File

@@ -12,3 +12,11 @@ export const formatFilterNumber = (num) => {
}
return num;
}
export const galleryImageUrl = (img) =>
img && img.replace("upload/", "upload/w_500/")
export const listingImageUrl = (img) =>
img && img.replace("upload/", "upload/w_205/")