Merge branch 'after-scraper-fix' into 'master'
Commented checkup out and Prostor fetch changed. See merge request saburly/marketalarm/web!102
This commit was merged in pull request #102.
This commit is contained in:
@@ -4,6 +4,7 @@ const fetch = require("../../helpers/fetchWrapper");
|
||||
const cheerio = require("cheerio");
|
||||
const moment = require("moment-timezone");
|
||||
const FormData = require("form-data");
|
||||
const nodeFetch = require("node-fetch");
|
||||
|
||||
const {
|
||||
AD_TYPE,
|
||||
@@ -197,7 +198,7 @@ class ProstorCrawler {
|
||||
|
||||
// console.log("[PROSTOR] Scraping : ", url);
|
||||
try {
|
||||
const adPageSource = await fetch(url, {
|
||||
const adPageSource = await nodeFetch(url, {
|
||||
headers: { Cookie: prostorCookie }
|
||||
});
|
||||
const body = await adPageSource.text();
|
||||
@@ -427,7 +428,7 @@ class ProstorCrawler {
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
const res = await nodeFetch(url, {
|
||||
headers: { Cookie: prostorCookie }
|
||||
});
|
||||
const body = await res.text();
|
||||
@@ -591,7 +592,7 @@ class ProstorCrawler {
|
||||
formData.append("email", PROSTOR_LOGIN.EMAIL);
|
||||
formData.append("password", PROSTOR_LOGIN.PASSWORD);
|
||||
|
||||
return fetch("https://prostor.ba/moj-prostor/prijava", {
|
||||
return nodeFetch("https://prostor.ba/moj-prostor/prijava", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
headers: { Cookie: prostorCookie }
|
||||
@@ -618,9 +619,12 @@ class ProstorCrawler {
|
||||
});
|
||||
}
|
||||
async getCookies() {
|
||||
const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
|
||||
headers: { Cookie: "" }
|
||||
});
|
||||
const getResponse = await nodeFetch(
|
||||
"https://prostor.ba/moj-prostor/prijava",
|
||||
{
|
||||
headers: { Cookie: "" }
|
||||
}
|
||||
);
|
||||
const raw = getResponse.headers.raw()["set-cookie"];
|
||||
const cookie = raw
|
||||
.map(datastring => {
|
||||
|
||||
@@ -136,7 +136,7 @@ const notifyRequestsWithDailyOption = async () => {
|
||||
};
|
||||
|
||||
const checkUpNotify = async () => {
|
||||
const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
|
||||
/* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
|
||||
|
||||
const asyncSendEmailActions = [];
|
||||
|
||||
@@ -153,7 +153,7 @@ const checkUpNotify = async () => {
|
||||
searchRequest.notifiedAt = new Date();
|
||||
searchRequest.save();
|
||||
}
|
||||
await Promise.all(asyncSendEmailActions);
|
||||
await Promise.all(asyncSendEmailActions);*/
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
||||
Reference in New Issue
Block a user