diff --git a/backend/helpers/externalSource.js b/backend/helpers/externalSource.js index 73fcff7..28236cc 100644 --- a/backend/helpers/externalSource.js +++ b/backend/helpers/externalSource.js @@ -1,6 +1,7 @@ let request = require ('request'); let Parser = require ('rss-parser'); -let summarizer = require('nodejs-text-summarizer') +let summarizer = require ('nodejs-text-summarizer'); +var htmlToText = require ('html-to-text'); const constants = require ('../config/constants'); let parser = new Parser (); @@ -43,7 +44,9 @@ module.exports = { .then (rawData => { let result = ''; rawData.forEach (post => { - result += post.title.rendered + ` `; + result += + post.title.rendered + + ` `; }); resolve (result); }) @@ -53,24 +56,35 @@ module.exports = { }); }, - getLatestNewsFromWP: function (sourceUrl, postCount = 10, includeTitle = false) { + getLatestNewsFromWP: function ( + sourceUrl, + postCount = 10, + includeTitle = false + ) { return new Promise ((resolve, reject) => { getDataFromWPJSON (sourceUrl, 1, postCount) .then (rawData => { let result = ''; - - try{ + + try { rawData.forEach (post => { - result += (includeTitle) ? post.title.rendered : ''; - result += (includeTitle) ? `` : ''; - result += summarizer(post.content.rendered); - result += ``; + result += includeTitle ? post.title.rendered : ''; + result += includeTitle + ? `` + : ''; + result += summarizer (post.content.rendered); + result += ``; }); - resolve(result); - }catch(err){ - reject(constants.voiceResponseString.ERROR_SUMMARIZING_CONTENT) + var textAnswer = htmlToText.fromString (result, { + wordwrap: false, + ignoreHref: true, + ignoreImage: true, + }); + + resolve (textAnswer); + } catch (err) { + reject (constants.voiceResponseString.ERROR_SUMMARIZING_CONTENT); } - }) .catch (err => { reject (constants.voiceResponseString.ERROR_FETCHING_CONTENT); diff --git a/backend/package-lock.json b/backend/package-lock.json index ebc0fa1..d29f472 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -34,14 +34,6 @@ "i18next-sprintf-postprocessor": "0.2.2" } }, - "apparatus": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.9.tgz", - "integrity": "sha1-N9zSWDStC2UQdllikduCPusZCL0=", - "requires": { - "sylvester": "0.0.21" - } - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -103,12 +95,6 @@ "tweetnacl": "0.14.5" } }, - "bindings": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz", - "integrity": "sha512-DpLh5EzMR2kzvX1KIlVC0VkC3iZtHKTgdtZ0a3pglBZdaQFjt5S9g9xd1lE+YvXyfd6mtCeRnrUfOLYiTMlNSw==", - "optional": true - }, "body-parser": { "version": "1.18.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", @@ -177,11 +163,6 @@ "delayed-stream": "1.0.0" } }, - "complex.js": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.0.4.tgz", - "integrity": "sha512-Syl95HpxUTS0QjwNxencZsKukgh1zdS9uXeXX2Us0pHaqBR6kiZZi0AkZ9VpZFwHJyVIUVzI4EumjWdXP3fy6w==" - }, "content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", @@ -241,11 +222,6 @@ "ms": "2.0.0" } }, - "decimal.js": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-9.0.1.tgz", - "integrity": "sha512-2h0iKbJwnImBk4TGk7CG1xadoA0g3LDPlQhQzbZ221zvG0p2YVUedbKIPsOZXKZGx6YmZMJKYOalpCMxSdDqTQ==" - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -261,6 +237,44 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "requires": { + "domelementtype": "1.1.3", + "entities": "1.1.1" + }, + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" + } + } + }, + "domelementtype": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", + "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=" + }, + "domhandler": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.1.tgz", + "integrity": "sha1-iS5HAAqZvlW783dP/qBWHYh5wlk=", + "requires": { + "domelementtype": "1.3.0" + } + }, + "domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "requires": { + "dom-serializer": "0.1.0", + "domelementtype": "1.3.0" + } + }, "ecc-jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", @@ -308,11 +322,6 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, - "escape-latex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.0.0.tgz", - "integrity": "sha512-oogO9Cg3n/4nspF34CTfXFymgI79skca66DebIIQgxVy6qRVqczl/ji2YGAqhFCzpD/oAt/fCWF4qlhMAfda+g==" - }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -433,11 +442,6 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, - "fraction.js": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.0.4.tgz", - "integrity": "sha512-aK/oGatyYLTtXRHjfEsytX5fieeR5H4s8sLorzcT12taFS+dbMZejnvm9gRa8mZAPwci24ucjq9epDyaq5u8Iw==" - }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", @@ -476,11 +480,40 @@ "sntp": "2.1.0" } }, + "he": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=" + }, "hoek": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.0.tgz", "integrity": "sha1-ctnQdU9/4lyi0BrY+PmpRJqJUm0=" }, + "html-to-text": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-4.0.0.tgz", + "integrity": "sha512-QQl5EEd97h6+3crtgBhkEAO6sQnZyDff8DAeJzoSkOc1Dqe1UvTUZER0B+KjBe6fPZqq549l2VUhtracus3ndA==", + "requires": { + "he": "1.1.1", + "htmlparser2": "3.9.2", + "lodash": "4.17.5", + "optimist": "0.6.1" + } + }, + "htmlparser2": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", + "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=", + "requires": { + "domelementtype": "1.3.0", + "domhandler": "2.4.1", + "domutils": "1.7.0", + "entities": "1.1.1", + "inherits": "2.0.3", + "readable-stream": "2.2.7" + } + }, "http-errors": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", @@ -561,11 +594,6 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, - "javascript-natural-sort": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", - "integrity": "sha1-+eIwPUUH9tdDVac2ZNFED7Wg71k=" - }, "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", @@ -608,21 +636,6 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==" }, - "mathjs": { - "version": "3.20.2", - "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-3.20.2.tgz", - "integrity": "sha512-3f6/+uf1cUtIz1rYFz775wekl/UEDSQ3mU6xdxW7qzpvvhc2v28i3UtLsGTRB+u8OqDWoSX6Dz8gehaGFs6tCA==", - "requires": { - "complex.js": "2.0.4", - "decimal.js": "9.0.1", - "escape-latex": "1.0.0", - "fraction.js": "4.0.4", - "javascript-natural-sort": "0.7.1", - "seed-random": "2.2.0", - "tiny-emitter": "2.0.2", - "typed-function": "0.10.7" - } - }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -656,6 +669,11 @@ "mime-db": "1.30.0" } }, + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" + }, "mongodb": { "version": "2.2.33", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.2.33.tgz", @@ -680,38 +698,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "nalapa": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/nalapa/-/nalapa-1.11.0.tgz", - "integrity": "sha1-YJZywk1fNZ73PQpmLyf11Kp8tj8=", - "requires": { - "lodash": "4.6.1" - }, - "dependencies": { - "lodash": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.6.1.tgz", - "integrity": "sha1-3wDBFkrSNrGDz8OIel6NOMxjy7w=" - } - } - }, - "nan": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", - "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==", - "optional": true - }, - "natural": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/natural/-/natural-0.5.6.tgz", - "integrity": "sha1-8hUrYfNr78b+Dta8gR+0mebp3M0=", - "requires": { - "apparatus": "0.0.9", - "sylvester": "0.0.21", - "underscore": "1.8.3", - "webworker-threads": "0.7.13" - } - }, "negotiator": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", @@ -726,16 +712,6 @@ "is-stream": "1.1.0" } }, - "nodejs-text-summarizer": { - "version": "github:GotPPay/nodejs-text-summarizer#9be60ffe387560b97f05c178be7db220012a52c1", - "requires": { - "lodash": "4.17.5", - "mathjs": "3.20.2", - "nalapa": "1.11.0", - "natural": "0.5.6", - "underscore.string": "3.3.4" - } - }, "nodemailer": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-4.6.3.tgz", @@ -754,6 +730,15 @@ "ee-first": "1.1.1" } }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "requires": { + "minimist": "0.0.10", + "wordwrap": "0.0.3" + } + }, "parseurl": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", @@ -890,11 +875,6 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, - "seed-random": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/seed-random/-/seed-random-2.2.0.tgz", - "integrity": "sha1-KpsZ4lCoFwmSMaW5mk2vgLf77VQ=" - }, "semver": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.4.1.tgz", @@ -951,11 +931,6 @@ "hoek": "4.2.0" } }, - "sprintf-js": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.1.tgz", - "integrity": "sha1-Nr54Mgr+WAH2zqPueLblqrlA6gw=" - }, "sshpk": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", @@ -989,16 +964,6 @@ "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" }, - "sylvester": { - "version": "0.0.21", - "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.21.tgz", - "integrity": "sha1-KYexzivS84sNzio0OIiEv6RADqc=" - }, - "tiny-emitter": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.0.2.tgz", - "integrity": "sha512-2NM0auVBGft5tee/OxP4PI3d8WItkDM+fPnaRAVo6xTDI2knbz9eC5ArWGqtGlYqiH3RU5yMpdyTTO7MguC4ow==" - }, "tough-cookie": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.3.tgz", @@ -1030,25 +995,6 @@ "mime-types": "2.1.17" } }, - "typed-function": { - "version": "0.10.7", - "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-0.10.7.tgz", - "integrity": "sha512-3mlZ5AwRMbLvUKkc8a1TI4RUJUS2H27pmD5q0lHRObgsoWzhDAX01yg82kwSP1FUw922/4Y9ZliIEh0qJZcz+g==" - }, - "underscore": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", - "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=" - }, - "underscore.string": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.3.4.tgz", - "integrity": "sha1-LCo/n4PmR2L9xF5s6sZRQoZCE9s=", - "requires": { - "sprintf-js": "1.1.1", - "util-deprecate": "1.0.2" - } - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -1100,21 +1046,16 @@ "extsprintf": "1.3.0" } }, - "webworker-threads": { - "version": "0.7.13", - "resolved": "https://registry.npmjs.org/webworker-threads/-/webworker-threads-0.7.13.tgz", - "integrity": "sha1-yEsYtrokElu503NC5E3rgVFi+4M=", - "optional": true, - "requires": { - "bindings": "1.3.0", - "nan": "2.10.0" - } - }, "whatwg-fetch": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz", "integrity": "sha1-nITsLc9oGH/wC8ZOEnS0QhduHIQ=" }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + }, "xml2js": { "version": "0.4.17", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.17.tgz", diff --git a/backend/package.json b/backend/package.json index 9e7b90c..7504d2d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,6 +8,7 @@ "body-parser": "^1.13.1", "ejs": "^2.5.7", "express": "^4.13.0", + "html-to-text": "^4.0.0", "isomorphic-fetch": "^2.2.1", "mongodb": "^2.2.33", "nodejs-text-summarizer": "GotPPay/nodejs-text-summarizer",