senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

sending notification when new items arrive with send grid

Browse Source
This commit is contained in:
egradanin
2019-01-19 19:20:21 +01:00
parent 18032c6ca6
commit ba2fcf0b8c
8 changed files with 540 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/MarketAlert.js
View File

@@ -10,8 +10,7 @@ const MarketAlert = sequelize.define("market_alert", {
last_date: Sequelize.STRING,
email: {
type: Sequelize.STRING,
allowNull: false,
unique: true
allowNull: false
}
});

68
backend/index.js
View File

@@ -1,68 +1,52 @@
let fetch = require("node-fetch");
let cheerio = require("cheerio");
let express = require("express");
const path = require("path");
const bodyParser = require("body-parser");
const MarketAlert = require("./MarketAlert");
const sendNotification = require("./utils/sendnotification");
const scrapTheItems = require("./utils/scraptheitems");
const sequelize = require("./db.js");
const app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
const port = process.env.PORT || 5000;
app.get("/api/sendnotifications", async function(req, res) {
let marketAlerts = await MarketAlert.findAll();
let lastDateUpdate = await Promise.all(
marketAlerts
.map(marketAlert => {
const { id, email, olx_url, last_date } = marketAlert.dataValues;
return { id, email, olx_url, last_date };
})
.map(sendNotification)
);
if (lastDateUpdate.some(dateUpdate => !dateUpdate)) return;
lastDateUpdate.length &&
lastDateUpdate.forEach(dateUpdate =>
MarketAlert.update(
{ last_date: dateUpdate.date },
{ where: { id: dateUpdate.id } }
)
);
});
app.get("/api/:url", async (req, res) => {
let url =
"https://www.olx.ba/pretraga?" +
req.params.url +
"&sort_order=desc&sort_po=datum";
let appts = [];
response = await fetch(url);
const body = await response.text();
const $ = cheerio.load(body);
$("#rezultatipretrage")
.find(".listitem")
.each(async (index, elem) => {
if (index == 0) {
lastItemDate = $(elem)
.find(".cijena > .datum > div")
.first()
.attr("data-cijelidatum");
}
const id = $(elem)
.find("a")
.first()
.attr("href");
const cijena = $(elem)
.find(".cijena > .datum > span")
.first()
.text();
const image = $(elem)
.find("a > .slika > img")
.first()
.attr("src");
appts.push({ url: id, price: cijena, image });
});
const [dan, mjesec, godina] = lastItemDate
.split(". u ")[0]
.split(".")
.map(el => Number(el));
const [sati, minute] = lastItemDate
.split(". u ")[1]
.split(":")
.map(el => Number(el));
last_date = String(new Date(godina, mjesec, dan, sati, minute));
let appts = scrapTheItems(url);
res.json({
last_date,
last_date: appts[0] && appts[0].date,
items: appts
});
});
app.post("/api/marketalerts", function(req, res) {
const { email, last_date, olx_url } = req.body;
sequelize.sync().then(() =>
MarketAlert.create({
olx_url,

416
backend/package-lock.json generated
View File

@@ -4,6 +4,47 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@sendgrid/client": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/@sendgrid/client/-/client-6.3.0.tgz",
"integrity": "sha512-fTy8vRpA9Whtf8ULQr/0vkSZaQvGQ97rY5N5PrevKRtugJMsJqFMKO0pwzEWeqITSg71aMMTj57QTgw3SjZvnQ==",
"requires": {
"@sendgrid/helpers": "^6.3.0",
"@types/request": "^2.0.3",
"request": "^2.81.0"
}
},
"@sendgrid/helpers": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/@sendgrid/helpers/-/helpers-6.3.0.tgz",
"integrity": "sha512-uTFcmhCDFg/2Uhz+z/cLwyLHH0UsblG49hKwdR7nKbWsGKWv4js7W32FlPdXqy2C/plTJ20vcPLgKM1m3F/MjQ==",
"requires": {
"chalk": "^2.0.1",
"deepmerge": "^2.1.1"
}
},
"@sendgrid/mail": {
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/@sendgrid/mail/-/mail-6.3.1.tgz",
"integrity": "sha512-5zIeAV9iU+0hQkrOQ/D4RB2MfpK+lNbOortIfQdCh95aMDF/TRc9WB8FGNhmQrx9YMuJTms5eiBklF0Fi/dbVg==",
"requires": {
"@sendgrid/client": "^6.3.0",
"@sendgrid/helpers": "^6.3.0"
}
},
"@types/caseless": {
"version": "0.12.1",
"resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz",
"integrity": "sha512-FhlMa34NHp9K5MY1Uz8yb+ZvuX0pnvn3jScRSNAb75KHGB8d3rEU6hqMs3Z2vjuytcMfRg6c5CHMc3wtYyD2/A=="
},
"@types/form-data": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz",
"integrity": "sha512-JAMFhOaHIciYVh8fb5/83nmuO/AHwmto+Hq7a9y8FzLDcC1KCU344XDOMEmahnrTFlHjgh4L0WJFczNIX2GxnQ==",
"requires": {
"@types/node": "*"
}
},
"@types/geojson": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.6.tgz",
@@ -14,6 +55,22 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz",
"integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ=="
},
"@types/request": {
"version": "2.48.1",
"resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz",
"integrity": "sha512-ZgEZ1TiD+KGA9LiAAPPJL68Id2UWfeSO62ijSXZjFJArVV+2pKcsVHmrcu+1oiE3q6eDGiFiSolRc4JHoerBBg==",
"requires": {
"@types/caseless": "*",
"@types/form-data": "*",
"@types/node": "*",
"@types/tough-cookie": "*"
}
},
"@types/tough-cookie": {
"version": "2.3.5",
"resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz",
"integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg=="
},
"accepts": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz",
@@ -23,11 +80,66 @@
"negotiator": "0.6.1"
}
},
"ajv": {
"version": "6.7.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz",
"integrity": "sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg==",
"requires": {
"fast-deep-equal": "^2.0.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"requires": {
"color-convert": "^1.9.0"
}
},
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"asn1": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
"integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
"requires": {
"safer-buffer": "~2.1.0"
}
},
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
},
"aws4": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ=="
},
"bcrypt-pbkdf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
"requires": {
"tweetnacl": "^0.14.3"
}
},
"bluebird": {
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
@@ -60,6 +172,21 @@
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"requires": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
}
},
"cheerio": {
"version": "1.0.0-rc.2",
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz",
@@ -82,6 +209,27 @@
"shimmer": "^1.1.0"
}
},
"color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"requires": {
"color-name": "1.1.3"
}
},
"color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
},
"combined-stream": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
"integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
"requires": {
"delayed-stream": "~1.0.0"
}
},
"content-disposition": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
@@ -102,6 +250,11 @@
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"css-select": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
@@ -118,6 +271,14 @@
"resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.2.tgz",
"integrity": "sha512-wan8dMWQ0GUeF7DGEPVjhHemVW/vy6xUYmFzRY8RYqgA0JtXC9rJmbScBjqSu6dg9q0lwPQy6ZAmJVr3PPTvqQ=="
},
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
"requires": {
"assert-plus": "^1.0.0"
}
},
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -126,6 +287,16 @@
"ms": "2.0.0"
}
},
"deepmerge": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.2.1.tgz",
"integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA=="
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"denque": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.3.0.tgz",
@@ -184,6 +355,15 @@
"resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.1.tgz",
"integrity": "sha512-ch5OQgvGDK2u8pSZeSYAQaV/lczImd7pMJ7BcEPXmnFVjy4yJIzP6CsODJUTH8mg1tyH1Z2abOiuJO3DjZ/GBw=="
},
"ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
"requires": {
"jsbn": "~0.1.0",
"safer-buffer": "^2.1.0"
}
},
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -204,6 +384,11 @@
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
},
"escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
},
"etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
@@ -246,6 +431,26 @@
"vary": "~1.1.2"
}
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
},
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
},
"fast-deep-equal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk="
},
"fast-json-stable-stringify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
"integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
},
"finalhandler": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz",
@@ -260,6 +465,21 @@
"unpipe": "~1.0.0"
}
},
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
},
"form-data": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
}
},
"forwarded": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
@@ -283,6 +503,33 @@
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.5.0.tgz",
"integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w=="
},
"getpass": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"requires": {
"assert-plus": "^1.0.0"
}
},
"har-schema": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
},
"har-validator": {
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
"integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
"requires": {
"ajv": "^6.5.5",
"har-schema": "^2.0.0"
}
},
"has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
},
"htmlparser2": {
"version": "3.10.0",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.0.tgz",
@@ -307,6 +554,16 @@
"statuses": ">= 1.4.0 < 2"
}
},
"http-signature": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
"requires": {
"assert-plus": "^1.0.0",
"jsprim": "^1.2.2",
"sshpk": "^1.7.0"
}
},
"iconv-lite": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
@@ -340,6 +597,47 @@
"resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
"integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ="
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
},
"isstream": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
},
"json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
},
"json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
},
"jsprim": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
"requires": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
"json-schema": "0.2.3",
"verror": "1.10.0"
}
},
"lodash": {
"version": "4.17.11",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
@@ -468,6 +766,11 @@
"boolbase": "~1.0.0"
}
},
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
},
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@@ -494,6 +797,11 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
},
"proxy-addr": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz",
@@ -508,6 +816,16 @@
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
},
"psl": {
"version": "1.1.31",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz",
"integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw=="
},
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
},
"qs": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
@@ -539,6 +857,33 @@
"util-deprecate": "^1.0.1"
}
},
"request": {
"version": "2.88.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
"integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
"requires": {
"aws-sign2": "~0.7.0",
"aws4": "^1.8.0",
"caseless": "~0.12.0",
"combined-stream": "~1.0.6",
"extend": "~3.0.2",
"forever-agent": "~0.6.1",
"form-data": "~2.3.2",
"har-validator": "~5.1.0",
"http-signature": "~1.2.0",
"is-typedarray": "~1.0.0",
"isstream": "~0.1.2",
"json-stringify-safe": "~5.0.1",
"mime-types": "~2.1.19",
"oauth-sign": "~0.9.0",
"performance-now": "^2.1.0",
"qs": "~6.5.2",
"safe-buffer": "^5.1.2",
"tough-cookie": "~2.4.3",
"tunnel-agent": "^0.6.0",
"uuid": "^3.3.2"
}
},
"retry-as-promised": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-2.3.2.tgz",
@@ -653,6 +998,22 @@
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
"integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
},
"sshpk": {
"version": "1.16.0",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.0.tgz",
"integrity": "sha512-Zhev35/y7hRMcID/upReIvRse+I9SVhyVre/KTJSJQWMz3C3+G+HpO7m1wK/yckEtujKZ7dS4hkVxAnmHaIGVQ==",
"requires": {
"asn1": "~0.2.3",
"assert-plus": "^1.0.0",
"bcrypt-pbkdf": "^1.0.0",
"dashdash": "^1.12.0",
"ecc-jsbn": "~0.1.1",
"getpass": "^0.1.1",
"jsbn": "~0.1.0",
"safer-buffer": "^2.0.2",
"tweetnacl": "~0.14.0"
}
},
"statuses": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz",
@@ -666,6 +1027,14 @@
"safe-buffer": "~5.1.0"
}
},
"supports-color": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
"requires": {
"has-flag": "^3.0.0"
}
},
"terraformer": {
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/terraformer/-/terraformer-1.0.9.tgz",
@@ -688,6 +1057,35 @@
"resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz",
"integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg="
},
"tough-cookie": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
"integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
"requires": {
"psl": "^1.1.24",
"punycode": "^1.4.1"
},
"dependencies": {
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
}
}
},
"tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"requires": {
"safe-buffer": "^5.0.1"
}
},
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
},
"type-is": {
"version": "1.6.16",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
@@ -702,6 +1100,14 @@
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"uri-js": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
"integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
"requires": {
"punycode": "^2.1.0"
}
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -727,6 +1133,16 @@
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
},
"verror": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"requires": {
"assert-plus": "^1.0.0",
"core-util-is": "1.0.2",
"extsprintf": "^1.2.0"
}
},
"wkx": {
"version": "0.4.6",
"resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.6.tgz",

1
backend/package.json
View File

@@ -9,6 +9,7 @@
"author": "Ehvan Gradanin",
"license": "ISC",
"dependencies": {
"@sendgrid/mail": "^6.3.1",
"cheerio": "^1.0.0-rc.2",
"express": "^4.16.4",
"mysql2": "^1.6.4",

8
backend/utils/arethereanynewitems.js Normal file
View File

@@ -0,0 +1,8 @@
const convertToDate = require("./convertToDate");
function areThereAnyNewItems(lastItemDate, controlDate) {
if (!lastItemDate) {
return true;
}
return new Date(controlDate) < convertToDate(lastItemDate);
}
module.exports = areThereAnyNewItems;

13
backend/utils/convertToDate.js Normal file
View File

@@ -0,0 +1,13 @@
function convertToDate(date) {
const [dan, mjesec, godina] = date
.split(". u ")[0]
.split(".")
.map(el => Number(el));
const [sati, minute] = date
.split(". u ")[1]
.split(":")
.map(el => Number(el));
return new Date(godina, mjesec, dan, sati, minute);
}
module.exports = convertToDate;

42
backend/utils/scraptheitems.js Normal file
View File

@@ -0,0 +1,42 @@
let fetch = require("node-fetch");
let cheerio = require("cheerio");
const areThereAnyNewItems = require("./arethereanynewitems");
async function scrapTheItems(url, controlDate, noNewItems = false) {
let items = [];
let response = await fetch(url);
const body = await response.text();
const $ = cheerio.load(body);
$("#rezultatipretrage")
.find(".listitem")
.each(async (index, elem) => {
if (noNewItems) return;
const itemDate = $(elem)
.find(".cijena > .datum > div")
.first()
.attr("data-cijelidatum");
if (controlDate && !areThereAnyNewItems(itemDate, controlDate)) {
noNewItems = true;
return;
}
const id = $(elem)
.find("a")
.first()
.attr("href");
const cijena = $(elem)
.find(".cijena > .datum > span")
.first()
.text();
const image = $(elem)
.find("a > .slika > img")
.first()
.attr("src");
items.push({ url: id, price: cijena, image, date: itemDate });
});
return items;
}
module.exports = scrapTheItems;

33
backend/utils/sendnotification.js Normal file
View File

@@ -0,0 +1,33 @@
const scrapTheItems = require("./scraptheitems");
const convertToDate = require("./convertToDate");
const sgMail = require("@sendgrid/mail");
// should be process.env.SENDGRID_API_KEY
sgMail.setApiKey(
"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"
);
async function sendNotification(marketAlert) {
const { id, email, olx_url, last_date } = marketAlert;
let url =
"https://www.olx.ba/pretraga?" + olx_url + "&sort_order=desc&sort_po=datum";
let newItems = await scrapTheItems(url);
let lastDate = newItems.length && newItems[0].date;
let message =
newItems.length &&
newItems.reduce(
(mes, item) => mes + `<strong>${item.url} i ${item.price}</strong>`,
""
);
const msg = {
to: email,
from: "test@example.com",
subject: "Market Alert",
text: "New items on olx",
html: message
};
if (message) {
await sgMail.send(msg);
return { id, date: String(convertToDate(lastDate)) };
}
}
module.exports = sendNotification;