From ba2fcf0b8c29bef6dfb131f1faf158c86b200743 Mon Sep 17 00:00:00 2001 From: egradanin Date: Sat, 19 Jan 2019 19:20:21 +0100 Subject: [PATCH] sending notification when new items arrive with send grid --- backend/MarketAlert.js | 3 +- backend/index.js | 68 ++--- backend/package-lock.json | 416 +++++++++++++++++++++++++++ backend/package.json | 1 + backend/utils/arethereanynewitems.js | 8 + backend/utils/convertToDate.js | 13 + backend/utils/scraptheitems.js | 42 +++ backend/utils/sendnotification.js | 33 +++ 8 files changed, 540 insertions(+), 44 deletions(-) create mode 100644 backend/utils/arethereanynewitems.js create mode 100644 backend/utils/convertToDate.js create mode 100644 backend/utils/scraptheitems.js create mode 100644 backend/utils/sendnotification.js diff --git a/backend/MarketAlert.js b/backend/MarketAlert.js index 7924077..3c800c8 100644 --- a/backend/MarketAlert.js +++ b/backend/MarketAlert.js @@ -10,8 +10,7 @@ const MarketAlert = sequelize.define("market_alert", { last_date: Sequelize.STRING, email: { type: Sequelize.STRING, - allowNull: false, - unique: true + allowNull: false } }); diff --git a/backend/index.js b/backend/index.js index 4c6ba36..252a8a8 100644 --- a/backend/index.js +++ b/backend/index.js @@ -1,68 +1,52 @@ -let fetch = require("node-fetch"); -let cheerio = require("cheerio"); let express = require("express"); const path = require("path"); const bodyParser = require("body-parser"); const MarketAlert = require("./MarketAlert"); +const sendNotification = require("./utils/sendnotification"); +const scrapTheItems = require("./utils/scraptheitems"); const sequelize = require("./db.js"); const app = express(); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); + const port = process.env.PORT || 5000; +app.get("/api/sendnotifications", async function(req, res) { + let marketAlerts = await MarketAlert.findAll(); + + let lastDateUpdate = await Promise.all( + marketAlerts + .map(marketAlert => { + const { id, email, olx_url, last_date } = marketAlert.dataValues; + return { id, email, olx_url, last_date }; + }) + .map(sendNotification) + ); + if (lastDateUpdate.some(dateUpdate => !dateUpdate)) return; + lastDateUpdate.length && + lastDateUpdate.forEach(dateUpdate => + MarketAlert.update( + { last_date: dateUpdate.date }, + { where: { id: dateUpdate.id } } + ) + ); +}); + app.get("/api/:url", async (req, res) => { let url = "https://www.olx.ba/pretraga?" + req.params.url + "&sort_order=desc&sort_po=datum"; - let appts = []; - response = await fetch(url); - const body = await response.text(); - - const $ = cheerio.load(body); - - $("#rezultatipretrage") - .find(".listitem") - .each(async (index, elem) => { - if (index == 0) { - lastItemDate = $(elem) - .find(".cijena > .datum > div") - .first() - .attr("data-cijelidatum"); - } - const id = $(elem) - .find("a") - .first() - .attr("href"); - const cijena = $(elem) - .find(".cijena > .datum > span") - .first() - .text(); - const image = $(elem) - .find("a > .slika > img") - .first() - .attr("src"); - appts.push({ url: id, price: cijena, image }); - }); - const [dan, mjesec, godina] = lastItemDate - .split(". u ")[0] - .split(".") - .map(el => Number(el)); - const [sati, minute] = lastItemDate - .split(". u ")[1] - .split(":") - .map(el => Number(el)); - last_date = String(new Date(godina, mjesec, dan, sati, minute)); + let appts = scrapTheItems(url); res.json({ - last_date, + last_date: appts[0] && appts[0].date, items: appts }); }); app.post("/api/marketalerts", function(req, res) { const { email, last_date, olx_url } = req.body; - sequelize.sync().then(() => MarketAlert.create({ olx_url, diff --git a/backend/package-lock.json b/backend/package-lock.json index 86b49d1..52a9248 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -4,6 +4,47 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@sendgrid/client": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@sendgrid/client/-/client-6.3.0.tgz", + "integrity": "sha512-fTy8vRpA9Whtf8ULQr/0vkSZaQvGQ97rY5N5PrevKRtugJMsJqFMKO0pwzEWeqITSg71aMMTj57QTgw3SjZvnQ==", + "requires": { + "@sendgrid/helpers": "^6.3.0", + "@types/request": "^2.0.3", + "request": "^2.81.0" + } + }, + "@sendgrid/helpers": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@sendgrid/helpers/-/helpers-6.3.0.tgz", + "integrity": "sha512-uTFcmhCDFg/2Uhz+z/cLwyLHH0UsblG49hKwdR7nKbWsGKWv4js7W32FlPdXqy2C/plTJ20vcPLgKM1m3F/MjQ==", + "requires": { + "chalk": "^2.0.1", + "deepmerge": "^2.1.1" + } + }, + "@sendgrid/mail": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@sendgrid/mail/-/mail-6.3.1.tgz", + "integrity": "sha512-5zIeAV9iU+0hQkrOQ/D4RB2MfpK+lNbOortIfQdCh95aMDF/TRc9WB8FGNhmQrx9YMuJTms5eiBklF0Fi/dbVg==", + "requires": { + "@sendgrid/client": "^6.3.0", + "@sendgrid/helpers": "^6.3.0" + } + }, + "@types/caseless": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz", + "integrity": "sha512-FhlMa34NHp9K5MY1Uz8yb+ZvuX0pnvn3jScRSNAb75KHGB8d3rEU6hqMs3Z2vjuytcMfRg6c5CHMc3wtYyD2/A==" + }, + "@types/form-data": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", + "integrity": "sha512-JAMFhOaHIciYVh8fb5/83nmuO/AHwmto+Hq7a9y8FzLDcC1KCU344XDOMEmahnrTFlHjgh4L0WJFczNIX2GxnQ==", + "requires": { + "@types/node": "*" + } + }, "@types/geojson": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.6.tgz", @@ -14,6 +55,22 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==" }, + "@types/request": { + "version": "2.48.1", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz", + "integrity": "sha512-ZgEZ1TiD+KGA9LiAAPPJL68Id2UWfeSO62ijSXZjFJArVV+2pKcsVHmrcu+1oiE3q6eDGiFiSolRc4JHoerBBg==", + "requires": { + "@types/caseless": "*", + "@types/form-data": "*", + "@types/node": "*", + "@types/tough-cookie": "*" + } + }, + "@types/tough-cookie": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", + "integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg==" + }, "accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", @@ -23,11 +80,66 @@ "negotiator": "0.6.1" } }, + "ajv": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz", + "integrity": "sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, "bluebird": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", @@ -60,6 +172,21 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, "cheerio": { "version": "1.0.0-rc.2", "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", @@ -82,6 +209,27 @@ "shimmer": "^1.1.0" } }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "combined-stream": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, "content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", @@ -102,6 +250,11 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, "css-select": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", @@ -118,6 +271,14 @@ "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.2.tgz", "integrity": "sha512-wan8dMWQ0GUeF7DGEPVjhHemVW/vy6xUYmFzRY8RYqgA0JtXC9rJmbScBjqSu6dg9q0lwPQy6ZAmJVr3PPTvqQ==" }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -126,6 +287,16 @@ "ms": "2.0.0" } }, + "deepmerge": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.2.1.tgz", + "integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA==" + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, "denque": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/denque/-/denque-1.3.0.tgz", @@ -184,6 +355,15 @@ "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.1.tgz", "integrity": "sha512-ch5OQgvGDK2u8pSZeSYAQaV/lczImd7pMJ7BcEPXmnFVjy4yJIzP6CsODJUTH8mg1tyH1Z2abOiuJO3DjZ/GBw==" }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -204,6 +384,11 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -246,6 +431,26 @@ "vary": "~1.1.2" } }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, "finalhandler": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", @@ -260,6 +465,21 @@ "unpipe": "~1.0.0" } }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", @@ -283,6 +503,33 @@ "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.5.0.tgz", "integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w==" }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + }, "htmlparser2": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.0.tgz", @@ -307,6 +554,16 @@ "statuses": ">= 1.4.0 < 2" } }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, "iconv-lite": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", @@ -340,6 +597,47 @@ "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, "lodash": { "version": "4.17.11", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", @@ -468,6 +766,11 @@ "boolbase": "~1.0.0" } }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -494,6 +797,11 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, "proxy-addr": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", @@ -508,6 +816,16 @@ "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, + "psl": { + "version": "1.1.31", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", + "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", @@ -539,6 +857,33 @@ "util-deprecate": "^1.0.1" } }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, "retry-as-promised": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-2.3.2.tgz", @@ -653,6 +998,22 @@ "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" }, + "sshpk": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.0.tgz", + "integrity": "sha512-Zhev35/y7hRMcID/upReIvRse+I9SVhyVre/KTJSJQWMz3C3+G+HpO7m1wK/yckEtujKZ7dS4hkVxAnmHaIGVQ==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, "statuses": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", @@ -666,6 +1027,14 @@ "safe-buffer": "~5.1.0" } }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + }, "terraformer": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/terraformer/-/terraformer-1.0.9.tgz", @@ -688,6 +1057,35 @@ "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg=" }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, "type-is": { "version": "1.6.16", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", @@ -702,6 +1100,14 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -727,6 +1133,16 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, "wkx": { "version": "0.4.6", "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.6.tgz", diff --git a/backend/package.json b/backend/package.json index 4657662..793dc4c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -9,6 +9,7 @@ "author": "Ehvan Gradanin", "license": "ISC", "dependencies": { + "@sendgrid/mail": "^6.3.1", "cheerio": "^1.0.0-rc.2", "express": "^4.16.4", "mysql2": "^1.6.4", diff --git a/backend/utils/arethereanynewitems.js b/backend/utils/arethereanynewitems.js new file mode 100644 index 0000000..39f0730 --- /dev/null +++ b/backend/utils/arethereanynewitems.js @@ -0,0 +1,8 @@ +const convertToDate = require("./convertToDate"); +function areThereAnyNewItems(lastItemDate, controlDate) { + if (!lastItemDate) { + return true; + } + return new Date(controlDate) < convertToDate(lastItemDate); +} +module.exports = areThereAnyNewItems; diff --git a/backend/utils/convertToDate.js b/backend/utils/convertToDate.js new file mode 100644 index 0000000..26da222 --- /dev/null +++ b/backend/utils/convertToDate.js @@ -0,0 +1,13 @@ +function convertToDate(date) { + const [dan, mjesec, godina] = date + .split(". u ")[0] + .split(".") + .map(el => Number(el)); + const [sati, minute] = date + .split(". u ")[1] + .split(":") + .map(el => Number(el)); + return new Date(godina, mjesec, dan, sati, minute); +} + +module.exports = convertToDate; diff --git a/backend/utils/scraptheitems.js b/backend/utils/scraptheitems.js new file mode 100644 index 0000000..75a9189 --- /dev/null +++ b/backend/utils/scraptheitems.js @@ -0,0 +1,42 @@ +let fetch = require("node-fetch"); +let cheerio = require("cheerio"); +const areThereAnyNewItems = require("./arethereanynewitems"); + +async function scrapTheItems(url, controlDate, noNewItems = false) { + let items = []; + let response = await fetch(url); + const body = await response.text(); + const $ = cheerio.load(body); + $("#rezultatipretrage") + .find(".listitem") + .each(async (index, elem) => { + if (noNewItems) return; + const itemDate = $(elem) + .find(".cijena > .datum > div") + .first() + .attr("data-cijelidatum"); + + if (controlDate && !areThereAnyNewItems(itemDate, controlDate)) { + noNewItems = true; + return; + } + + const id = $(elem) + .find("a") + .first() + .attr("href"); + const cijena = $(elem) + .find(".cijena > .datum > span") + .first() + .text(); + const image = $(elem) + .find("a > .slika > img") + .first() + .attr("src"); + + items.push({ url: id, price: cijena, image, date: itemDate }); + }); + return items; +} + +module.exports = scrapTheItems; diff --git a/backend/utils/sendnotification.js b/backend/utils/sendnotification.js new file mode 100644 index 0000000..be3251c --- /dev/null +++ b/backend/utils/sendnotification.js @@ -0,0 +1,33 @@ +const scrapTheItems = require("./scraptheitems"); +const convertToDate = require("./convertToDate"); +const sgMail = require("@sendgrid/mail"); +// should be process.env.SENDGRID_API_KEY +sgMail.setApiKey( + "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" +); + +async function sendNotification(marketAlert) { + const { id, email, olx_url, last_date } = marketAlert; + let url = + "https://www.olx.ba/pretraga?" + olx_url + "&sort_order=desc&sort_po=datum"; + let newItems = await scrapTheItems(url); + let lastDate = newItems.length && newItems[0].date; + let message = + newItems.length && + newItems.reduce( + (mes, item) => mes + `${item.url} i ${item.price}`, + "" + ); + const msg = { + to: email, + from: "test@example.com", + subject: "Market Alert", + text: "New items on olx", + html: message + }; + if (message) { + await sgMail.send(msg); + return { id, date: String(convertToDate(lastDate)) }; + } +} +module.exports = sendNotification;