senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Olx scraper debugging.

Browse Source
This commit is contained in:
Naida Vatric
2020-02-28 15:41:20 +01:00
parent 747f56941a
commit 90e171d07d
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/crawler/specificCrawlers/olx.js
View File

@@ -205,7 +205,7 @@ class OlxCrawler {
} }
async scrapeAd(url) { async scrapeAd(url) {
//console.log("Scraping : ", url); console.log("Scraping : ", url);
try { try {
const adPageSource = await fetch(url); const adPageSource = await fetch(url);
const body = await adPageSource.text(); const body = await adPageSource.text();
@@ -249,6 +249,8 @@ class OlxCrawler {
.text() .text()
.trim(); .trim();
// //
console.log("Title:", title);
console.log("Url scraped:", url);
console.log("Normal price value:", normalPriceValue); console.log("Normal price value:", normalPriceValue);
console.log("Urgent price value:", urgentPriceValue); console.log("Urgent price value:", urgentPriceValue);
// //
@@ -293,7 +295,7 @@ class OlxCrawler {
let otherInformationDivId; let otherInformationDivId;
//We need to locate DIV ID where other information are stored //We need to locate DIV ID where other information are stored
for (let possibleId = 10; possibleId <= 20; possibleId++) { for (let possibleId = 1; possibleId <= 30; possibleId++) {
const adTypeFieldTitle = $( const adTypeFieldTitle = $(
`#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1` `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
) )

2
test/olxScrapeTest.js
View File

@@ -9,7 +9,7 @@ if (urlToScrape) {
(async () => { (async () => {
const data = await crawler.scrapeAd(urlToScrape); const data = await crawler.scrapeAd(urlToScrape);
console.log(data); console.log("Scraped data:", data);
})(); })();
} else { } else {
console.log("No URL to scrape. Use like this : "); console.log("No URL to scrape. Use like this : ");