Use cloudinary

crawler/package.json

@@ -11,6 +11,7 @@
     "babel-polyfill": "^6.16.0",
     "babel-preset-es2015": "^6.18.0",
     "cheerio": "^0.22.0",
+    "cloudinary": "^1.8.0",
     "dotenv": "^2.0.0",
     "fetch": "^1.1.0",
     "json-loader": "^0.5.4",

crawler/savers/mongo.js

@@ -33,8 +33,9 @@ export default class MongoSaver {
       return results[key]
     });
 
-    this.collection.update({ url: results.url }, resultsForMongo, { upsert: true });
+    for(const doc of resultsForMongo) {
-    // this.collection.insert(resultsForMongo);
+      this.collection.update({ url: doc.url }, doc, { upsert: true });
+    }
   }
 
   async close() {

crawler/specific/olx.js

@@ -3,6 +3,7 @@
 let fetch = require('node-fetch');
 let cheerio = require('cheerio');
 let fs = require('fs');
+let cloudinary = require('cloudinary');
 
 import {
   AD_TYPE_SALE,
@@ -68,6 +69,14 @@ export default class OlxCrawler {
         images.push(img);
       }
 
+      const uploadPromises = images.map(img => {
+        const imgFixed = eval(`'${img}'`);
+        return cloudinary.uploader.upload(eval(`'${img}'`));
+      });
+
+      const uploadResults = await Promise.all(uploadPromises);
+      const cloudinaryImages = uploadResults.map(ur => ur.url);
+
       if (matches && matches.length >= 3) {
         lat = matches[1];
         lng = matches[2];
@@ -99,12 +108,12 @@ export default class OlxCrawler {
         lat,
         lng,
         loc: [parseFloat(lat), parseFloat(lng)],
-        images
+        images: cloudinaryImages
       };
 
       return data;
     } catch (e) {
-      console.error('Exception caught: ' + e);
+      console.error('Exception caught: ' + e.message);
     }
 
     return null;

crawler/yarn.lock

@@ -745,6 +745,13 @@ clone@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.2.tgz#260b7a99ebb1edfe247538175f783243cb19d149"
 
+cloudinary:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/cloudinary/-/cloudinary-1.8.0.tgz#52e7a18dfe3c0ece01abc71341be95d0ea198d85"
+  dependencies:
+    lodash "3.10.x"
+    q "1.4.x"
+
 code-point-at@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
@@ -1532,6 +1539,10 @@ lodash@^4.14.0, lodash@^4.2.0:
   version "4.16.6"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.6.tgz#d22c9ac660288f3843e16ba7d2b5d06cca27d777"
 
+lodash@3.10.x:
+  version "3.10.1"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
+
 longest@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097"
@@ -1860,6 +1871,10 @@ q@0.9.7:
   version "0.9.7"
   resolved "https://registry.yarnpkg.com/q/-/q-0.9.7.tgz#4de2e6cb3b29088c9e4cbc03bf9d42fb96ce2f75"
 
+q@1.4.x:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/q/-/q-1.4.1.tgz#55705bcd93c5f3673530c2c2cbc0c2b3addc286e"
+
 qs@~6.2.0:
   version "6.2.1"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.2.1.tgz#ce03c5ff0935bc1d9d69a9f14cbd18e568d67625"

web/components/Gallery.js

@@ -1,4 +1,5 @@
 import React from 'react';
+import {galleryImageUrl} from '../lib/helpers';
 
 export default class Gallery extends React.Component {
   onPrevClick (e) {
@@ -24,7 +25,7 @@ export default class Gallery extends React.Component {
 
     return (
       <div className="ld-image-container">
-        <img src={images[imageIndex]}></img>
+        <img src={galleryImageUrl(images[imageIndex])}></img>
         {showPrev ?
           <div
             className='prev-button'

web/components/Listings.js

@@ -1,6 +1,6 @@
 import React from 'react';
 import {findDOMNode} from 'react-dom';
-import {formatPrice} from '../lib/helpers';
+import {formatPrice, listingImageUrl} from '../lib/helpers';
 import {loadListing} from '../lib/api';
 
 export default class Listings extends React.Component {
@@ -22,8 +22,6 @@ export default class Listings extends React.Component {
   }
 
   onListingClick(id) {
-
-
     loadListing(id).then(l => l.text()).then(l => {
       this.props.dispatch({type: 'UPDATE_ROUTE', action: {
         toDispatch: {
@@ -79,7 +77,7 @@ export default class Listings extends React.Component {
           className="property-list-item"
           onClick={this.onListingClick.bind(this, l._id)}>
           <div className="pli-image">
-            <img src={images[0]} alt=""></img>
+            <img src={listingImageUrl(images[0])} alt=""></img>
           </div>
           <div className="pli-details">
             <div className="price">{formatPrice(l.price)}</div>

web/lib/helpers.js

@@ -12,3 +12,11 @@ export const formatFilterNumber = (num) => {
   }
   return num;
 }
+
+export const galleryImageUrl = (img) =>
+  img && img.replace("upload/", "upload/w_500/")
+
+
+export const listingImageUrl = (img) =>
+  img && img.replace("upload/", "upload/w_205/")
+