senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Add caching to fetch wrapper

Browse Source
This commit is contained in:
Senad Uka
2020-10-21 06:16:35 +02:00
parent 88f9d10586
commit 92e4f4ed5a
7 changed files with 38 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/crawler/specificCrawlers/aktido.js
View File

@@ -159,7 +159,7 @@ class AktidoCrawler {
}
try {
const res = await fetch(url);
const res = await fetch(url, {}, false);
const body = await res.text();
const $ = cheerio.load(body);
let hrefs = [];

2
app/crawler/specificCrawlers/olx.js
View File

@@ -217,7 +217,7 @@ class OlxCrawler {
}
try {
const res = await fetch(url);
const res = await fetch(url, {}, false);
logDebug("Got category results for: ", url);
const body = await res.text();
logDebug("Got category results text for: ", url);

2
app/crawler/specificCrawlers/rental.js
View File

@@ -159,7 +159,7 @@ class RentalCrawler {
}
try {
const res = await fetch(url);
const res = await fetch(url, {} , false);
const body = await res.text();
const $ = cheerio.load(body);
let hrefs = [];

2
app/crawler/specificCrawlers/saljic.js
View File

@@ -160,7 +160,7 @@ class SaljicCrawler {
}
try {
const res = await fetch(url);
const res = await fetch(url, {}, false);
const body = await res.text();
const $ = cheerio.load(body);
let hrefs = [];