senaduka/old-svevijesti
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix scrapers

Browse Source
This commit is contained in:
Senad Uka
2022-02-17 18:58:16 +01:00
parent f4ea31373d
commit 5f22302e44
3 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/scraper/blijesak.go
View File

@@ -80,7 +80,7 @@ func setupBljesakArticlePageCrawler(crArticlePage *colly.Collector) {
text := ""
e.ForEach("article.b-article-detail, div#infiniteLoadBreakpoint, div.col-article-content, div.intro, div.s-main-content", func(_ int, el *colly.HTMLElement) {
e.ForEach("div.col-xs-12, article.b-article-detail, div#infiniteLoadBreakpoint, div.col-article-content, div.intro, div.s-main-content, h2, h3, h4, h5", func(_ int, el *colly.HTMLElement) {
text += extractJustText(el.DOM)
})

2
internal/scraper/scraper.go
View File

@@ -17,7 +17,7 @@ func extractJustText(el *goquery.Selection) string {
}
el.Children().Each(func(_ int, el2 *goquery.Selection) {
if el2.Is("div, p, span, a") {
if el2.Is("div, p, span, a, h2, h3, h4, b, i") {
textPart += extractJustText(el2)
}
})

2
internal/scraper/srpskainfo.go
View File

@@ -79,7 +79,7 @@ func setupSiArticlePageCrawler(crArticlePage *colly.Collector) {
text := ""
e.ForEach("p.article__top-content, p.article__content, h4.article__content, h3.article__content, h2.article__content, div.article__content", func(_ int, el *colly.HTMLElement) {
e.ForEach("div.article__top-content, div.article__content, h4, h3, h2, div.article__content", func(_ int, el *colly.HTMLElement) {
text += extractJustText(el.DOM)
})