Merge branch 'after-scraper-fix' into 'master'

Commented checkup out and Prostor fetch changed.

See merge request saburly/marketalarm/web!102
This commit was merged in pull request #102.
This commit is contained in:
Naida Vatric
2020-02-27 20:31:14 +00:00
2 changed files with 12 additions and 8 deletions

View File

@@ -4,6 +4,7 @@ const fetch = require("../../helpers/fetchWrapper");
const cheerio = require("cheerio");
const moment = require("moment-timezone");
const FormData = require("form-data");
const nodeFetch = require("node-fetch");
const {
AD_TYPE,
@@ -197,7 +198,7 @@ class ProstorCrawler {
// console.log("[PROSTOR] Scraping : ", url);
try {
const adPageSource = await fetch(url, {
const adPageSource = await nodeFetch(url, {
headers: { Cookie: prostorCookie }
});
const body = await adPageSource.text();
@@ -427,7 +428,7 @@ class ProstorCrawler {
}
try {
const res = await fetch(url, {
const res = await nodeFetch(url, {
headers: { Cookie: prostorCookie }
});
const body = await res.text();
@@ -591,7 +592,7 @@ class ProstorCrawler {
formData.append("email", PROSTOR_LOGIN.EMAIL);
formData.append("password", PROSTOR_LOGIN.PASSWORD);
return fetch("https://prostor.ba/moj-prostor/prijava", {
return nodeFetch("https://prostor.ba/moj-prostor/prijava", {
method: "POST",
body: formData,
headers: { Cookie: prostorCookie }
@@ -618,9 +619,12 @@ class ProstorCrawler {
});
}
async getCookies() {
const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
headers: { Cookie: "" }
});
const getResponse = await nodeFetch(
"https://prostor.ba/moj-prostor/prijava",
{
headers: { Cookie: "" }
}
);
const raw = getResponse.headers.raw()["set-cookie"];
const cookie = raw
.map(datastring => {

View File

@@ -136,7 +136,7 @@ const notifyRequestsWithDailyOption = async () => {
};
const checkUpNotify = async () => {
const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
/* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
const asyncSendEmailActions = [];
@@ -153,7 +153,7 @@ const checkUpNotify = async () => {
searchRequest.notifiedAt = new Date();
searchRequest.save();
}
await Promise.all(asyncSendEmailActions);
await Promise.all(asyncSendEmailActions);*/
};
module.exports = {