senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

add debugging log switch for crawler process

Browse Source
This commit is contained in:
Bilal Catic
2019-10-25 11:08:52 +02:00
parent 7e3b0bfcd5
commit 747ebb88e5
5 changed files with 25 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/crawler/specificCrawlers/olx.js
View File

@@ -13,7 +13,10 @@ const {
CRAWLER_AD_TYPE
} = require("../../common/enums");
const { DEFAULT_TIMEZONE } = require("../../config/appConfig");
const {
DEFAULT_TIMEZONE,
PRINT_CRAWLER_DEBUG
} = require("../../config/appConfig");
const OLX_ENUMS = {
OLX_AD_TYPE: {
@@ -156,6 +159,10 @@ class OlxCrawler {
}
async indexSinglePage(url, maxResultsPerPage) {
if (PRINT_CRAWLER_DEBUG) {
console.log("[OLX] Index page : ", url);
}
try {
const res = await fetch(url);
const body = await res.text();

6
app/crawler/specificCrawlers/prostor.js
View File

@@ -11,6 +11,8 @@ const {
CRAWLER_AD_TYPE
} = require("../../common/enums");
const { PRINT_CRAWLER_DEBUG } = require("../../config/appConfig");
const PROSTOR_ENUMS = {
PROSTOR_AD_TYPE: {
[CRAWLER_AD_TYPE.ALL]: "&action=0",
@@ -78,7 +80,9 @@ class ProstorCrawler {
}
async extractRealEstates(url) {
console.log("[PROSTOR] Index page : ", url);
if (PRINT_CRAWLER_DEBUG) {
console.log("[PROSTOR] Index page : ", url);
}
try {
const res = await fetch(url);

9
app/crawler/specificCrawlers/rental.js
View File

@@ -14,7 +14,10 @@ const {
CRAWLER_AD_TYPE
} = require("../../common/enums");
const { DEFAULT_TIMEZONE } = require("../../config/appConfig");
const {
DEFAULT_TIMEZONE,
PRINT_CRAWLER_DEBUG
} = require("../../config/appConfig");
const RENTAL_ENUMS = {
RENTAL_AD_TYPE: {
@@ -159,7 +162,9 @@ class RentalCrawler {
}
async indexSinglePage(url, maxResultsPerPage) {
// console.log("[RENTAL] Index page : ", url);
if (PRINT_CRAWLER_DEBUG) {
console.log("[RENTAL] Index page : ", url);
}
try {
const res = await fetch(url);