diff --git a/crawler/package.json b/crawler/package.json index b27a87d..031f21e 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -1,32 +1,33 @@ { - "name": "stan", - "version": "1.0.0", - "description": "", - "main": "index.js", - "dependencies": { - "babel": "^6.5.2", - "babel-core": "^6.18.2", - "babel-loader": "^6.2.7", - "babel-plugin-transform-async-to-generator": "^6.16.0", - "babel-polyfill": "^6.16.0", - "babel-preset-es2015": "^6.18.0", - "cheerio": "^0.22.0", - "dotenv": "^2.0.0", - "fetch": "^1.1.0", - "json-loader": "^0.5.4", - "mongodb": "^2.2.11", - "node-fetch": "^1.6.3", - "source-map-support": "^0.4.6", - "twilio": "^2.11.0" - }, - "devDependencies": { - "webpack": "^1.13.3" - }, - "scripts": { - "dev": "webpack", - "prod": "webpack -p", - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "ISC" + "name": "stan", + "version": "1.0.0", + "description": "", + "main": "index.js", + "dependencies": { + "babel": "^6.5.2", + "babel-core": "^6.18.2", + "babel-loader": "^6.2.7", + "babel-plugin-transform-async-to-generator": "^6.16.0", + "babel-polyfill": "^6.16.0", + "babel-preset-es2015": "^6.18.0", + "cheerio": "^0.22.0", + "cloudinary": "^1.8.0", + "dotenv": "^2.0.0", + "fetch": "^1.1.0", + "json-loader": "^0.5.4", + "mongodb": "^2.2.11", + "node-fetch": "^1.6.3", + "source-map-support": "^0.4.6", + "twilio": "^2.11.0" + }, + "devDependencies": { + "webpack": "^1.13.3" + }, + "scripts": { + "dev": "webpack", + "prod": "webpack -p", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC" } diff --git a/crawler/savers/mongo.js b/crawler/savers/mongo.js index 2bff4ee..582905e 100644 --- a/crawler/savers/mongo.js +++ b/crawler/savers/mongo.js @@ -33,8 +33,9 @@ export default class MongoSaver { return results[key] }); - this.collection.update({ url: results.url }, resultsForMongo, { upsert: true }); - // this.collection.insert(resultsForMongo); + for(const doc of resultsForMongo) { + this.collection.update({ url: doc.url }, doc, { upsert: true }); + } } async close() { diff --git a/crawler/specific/olx.js b/crawler/specific/olx.js index 02a195e..76ddf34 100644 --- a/crawler/specific/olx.js +++ b/crawler/specific/olx.js @@ -3,6 +3,7 @@ let fetch = require('node-fetch'); let cheerio = require('cheerio'); let fs = require('fs'); +let cloudinary = require('cloudinary'); import { AD_TYPE_SALE, @@ -68,6 +69,14 @@ export default class OlxCrawler { images.push(img); } + const uploadPromises = images.map(img => { + const imgFixed = eval(`'${img}'`); + return cloudinary.uploader.upload(eval(`'${img}'`)); + }); + + const uploadResults = await Promise.all(uploadPromises); + const cloudinaryImages = uploadResults.map(ur => ur.url); + if (matches && matches.length >= 3) { lat = matches[1]; lng = matches[2]; @@ -99,12 +108,12 @@ export default class OlxCrawler { lat, lng, loc: [parseFloat(lat), parseFloat(lng)], - images + images: cloudinaryImages }; return data; } catch (e) { - console.error('Exception caught: ' + e); + console.error('Exception caught: ' + e.message); } return null; diff --git a/crawler/yarn.lock b/crawler/yarn.lock index 98fec16..b515184 100644 --- a/crawler/yarn.lock +++ b/crawler/yarn.lock @@ -745,6 +745,13 @@ clone@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.2.tgz#260b7a99ebb1edfe247538175f783243cb19d149" +cloudinary: + version "1.8.0" + resolved "https://registry.yarnpkg.com/cloudinary/-/cloudinary-1.8.0.tgz#52e7a18dfe3c0ece01abc71341be95d0ea198d85" + dependencies: + lodash "3.10.x" + q "1.4.x" + code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" @@ -1532,6 +1539,10 @@ lodash@^4.14.0, lodash@^4.2.0: version "4.16.6" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.6.tgz#d22c9ac660288f3843e16ba7d2b5d06cca27d777" +lodash@3.10.x: + version "3.10.1" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6" + longest@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" @@ -1860,6 +1871,10 @@ q@0.9.7: version "0.9.7" resolved "https://registry.yarnpkg.com/q/-/q-0.9.7.tgz#4de2e6cb3b29088c9e4cbc03bf9d42fb96ce2f75" +q@1.4.x: + version "1.4.1" + resolved "https://registry.yarnpkg.com/q/-/q-1.4.1.tgz#55705bcd93c5f3673530c2c2cbc0c2b3addc286e" + qs@~6.2.0: version "6.2.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.2.1.tgz#ce03c5ff0935bc1d9d69a9f14cbd18e568d67625" diff --git a/web/components/Gallery.js b/web/components/Gallery.js index 4a3b0a3..0dd6a5f 100644 --- a/web/components/Gallery.js +++ b/web/components/Gallery.js @@ -1,4 +1,5 @@ import React from 'react'; +import {galleryImageUrl} from '../lib/helpers'; export default class Gallery extends React.Component { onPrevClick (e) { @@ -24,7 +25,7 @@ export default class Gallery extends React.Component { return (
- + {showPrev ?
l.text()).then(l => { this.props.dispatch({type: 'UPDATE_ROUTE', action: { toDispatch: { @@ -79,7 +77,7 @@ export default class Listings extends React.Component { className="property-list-item" onClick={this.onListingClick.bind(this, l._id)}>
- +
{formatPrice(l.price)}
diff --git a/web/lib/helpers.js b/web/lib/helpers.js index 7fe59a1..c3948f8 100644 --- a/web/lib/helpers.js +++ b/web/lib/helpers.js @@ -12,3 +12,11 @@ export const formatFilterNumber = (num) => { } return num; } + +export const galleryImageUrl = (img) => + img && img.replace("upload/", "upload/w_500/") + + +export const listingImageUrl = (img) => + img && img.replace("upload/", "upload/w_205/") +