Merge branch 'remove-obsolete-files' into 'master'

Remove obsolete files

See merge request saburly/marketalarm/web!52

app/lib/areThereAnyNewItems.js

@@ -1,10 +0,0 @@
-const convertToDate = require("./convertToDate");
-
-function areThereAnyNewItems(lastItemDate, controlDate) {
-  if (!lastItemDate) {
-    return true;
-  }
-  return new Date(controlDate) < convertToDate(lastItemDate);
-}
-
-module.exports = areThereAnyNewItems;

app/lib/convertToDate.js

@@ -1,13 +0,0 @@
-function convertToDate(date) {
-  const [dan, mjesec, godina] = date
-    .split(". u ")[0]
-    .split(".")
-    .map(el => Number(el));
-  const [sati, minute] = date
-    .split(". u ")[1]
-    .split(":")
-    .map(el => Number(el));
-  return new Date(godina, mjesec, dan, sati, minute);
-}
-
-module.exports = convertToDate;

app/lib/scrapTheItems.js

@@ -1,42 +0,0 @@
-let fetch = require("node-fetch");
-let cheerio = require("cheerio");
-const areThereAnyNewItems = require("./areThereAnyNewItems");
-
-async function scrapTheItems(url, controlDate, noNewItems = false) {
-  let items = [];
-  let response = await fetch(url);
-  const body = await response.text();
-  const $ = cheerio.load(body);
-  $("#rezultatipretrage")
-    .find(".listitem")
-    .each(async (index, elem) => {
-      if (noNewItems) return;
-      const itemDate = $(elem)
-        .find(".cijena > .datum > div")
-        .first()
-        .attr("data-cijelidatum");
-
-      if (controlDate && !areThereAnyNewItems(itemDate, controlDate)) {
-        noNewItems = true;
-        return;
-      }
-
-      const id = $(elem)
-        .find("a")
-        .first()
-        .attr("href");
-      const cijena = $(elem)
-        .find(".cijena > .datum > span")
-        .first()
-        .text();
-      const image = $(elem)
-        .find("a > .slika > img")
-        .first()
-        .attr("src");
-
-      items.push({ url: id, price: cijena, image, date: itemDate });
-    });
-  return items;
-}
-
-module.exports = scrapTheItems;

app/lib/sendNotification.js

@@ -1,57 +0,0 @@
-const scrapTheItems = require("./scrapTheItems");
-const convertToDate = require("./convertToDate");
-const AWS = require("aws-sdk");
-// AWS.config.update({region: 'eu-central-1'});
-
-async function sendNotification(marketAlert) {
-  const { id, email, olx_url } = marketAlert;
-  let url =
-    "https://www.olx.ba/pretraga?" + olx_url + "&sort_order=desc&sort_po=datum";
-  let newItems = await scrapTheItems(url);
-  let lastDate = newItems.length && newItems[0].date;
-  let message =
-    newItems.length &&
-    newItems.reduce(
-      (mes, item) => mes + `<strong>${item.url} i  ${item.price}</strong>`,
-      ""
-    );
-
-  // Create sendEmail params
-  const params = {
-    Destination: {
-      /* required */
-      CcAddresses: [],
-      ToAddresses: [email]
-    },
-    Message: {
-      /* required */
-      Body: {
-        /* required */
-        Html: {
-          Charset: "UTF-8",
-          Data: message
-        },
-        Text: {
-          Charset: "UTF-8",
-          Data: message // TODO: convert to text
-        }
-      },
-      Subject: {
-        Charset: "UTF-8",
-        Data: "Javimi alert"
-      }
-    },
-    Source: 'Kivi.ba <info@saburly.com>' /* required */,
-    ReplyToAddresses: ["info@saburly.com"]
-  };
-
-  if (message) {
-    const sendPromise = new AWS.SES({ apiVersion: "2010-12-01" })
-      .sendEmail(params)
-      .promise();
-    await sendPromise;
-    return { id, date: String(convertToDate(lastDate)) };
-  }
-}
-
-module.exports = sendNotification;

package.json

@@ -12,7 +12,8 @@
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",
     "crawl": "cd app/crawler && node npmCrawl.js",
-    "test-search": "cd test && node searchTest.js"
+    "test-search": "cd test && node searchTest.js",
+    "test-olx-scraper": "cd test && node olxScrapeTest.js"
   },
   "repository": {
     "type": "git",

test/olxScrapeTest.js

@@ -0,0 +1,17 @@
+"use strict";
+
+const olxCrawler = require("../app/crawler/specific/olx");
+
+const urlToScrape = process.argv[2] || undefined;
+
+if (urlToScrape) {
+  const crawler = new olxCrawler();
+
+  (async () => {
+    const data = await crawler.scrapeAd(urlToScrape);
+    console.log(data);
+  })();
+} else {
+  console.log("No URL to scrape. Use like this : ");
+  console.log("npm run test-olx-scraper -- URL_TO_SCRAPE");
+}