105 lines
2.7 KiB
Go
105 lines
2.7 KiB
Go
package scraper
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/gocolly/colly"
|
|
"github.com/gosimple/slug"
|
|
"gitlab.com/kbr4/svevijesti/internal/model"
|
|
"math/rand"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
var AvazArticles = make(chan model.ScrapedArticle)
|
|
var AvazCandidates = make(chan string)
|
|
var AvazApprovedSites = make(chan string, 2)
|
|
|
|
func CrawlAvaz() {
|
|
|
|
crHomePage := colly.NewCollector(colly.AllowedDomains("avaz.ba"))
|
|
crArticlePage := colly.NewCollector(colly.AllowedDomains("avaz.ba"))
|
|
|
|
setupAvazArticlePageCrawler(crArticlePage)
|
|
setupAvazHomepageCrawler(crHomePage, crArticlePage)
|
|
|
|
go visitAvazApprovedPages(crArticlePage)
|
|
}
|
|
|
|
func visitAvazApprovedPages(crArticlePage *colly.Collector) {
|
|
fmt.Println("Consuming sites!")
|
|
for url := range AvazApprovedSites {
|
|
fmt.Println("Visiting: ", url)
|
|
crArticlePage.Visit(url)
|
|
}
|
|
}
|
|
|
|
func setupAvazHomepageCrawler(crHomePage *colly.Collector, crArticlePage *colly.Collector) {
|
|
|
|
articleUrlR, _ := regexp.Compile("/\\d\\d+/([a-z0-9-]+)")
|
|
articleBlacklist, _ := regexp.Compile("(info-vodic|foto-data)")
|
|
crHomePage.OnHTML("a", func(e *colly.HTMLElement) {
|
|
url := e.Attr("href")
|
|
completeUrl := url
|
|
if articleUrlR.MatchString(url) && !articleBlacklist.MatchString(url) {
|
|
AvazCandidates <- completeUrl
|
|
}
|
|
})
|
|
|
|
crHomePage.OnScraped(func(_ *colly.Response) {
|
|
time.Sleep(5 * time.Second)
|
|
terminating := model.ScrapedArticle{}
|
|
terminating.Title = model.Terminator
|
|
AvazArticles <- terminating
|
|
AvazApprovedSites <- model.Terminator
|
|
AvazCandidates <- model.Terminator
|
|
})
|
|
|
|
crHomePage.OnError(func(_ *colly.Response, _ error) {
|
|
time.Sleep(5 * time.Second)
|
|
terminating := model.ScrapedArticle{}
|
|
terminating.Title = model.Terminator
|
|
AvazArticles <- terminating
|
|
AvazApprovedSites <- model.Terminator
|
|
AvazCandidates <- model.Terminator
|
|
})
|
|
|
|
go crHomePage.Visit("https://avaz.ba")
|
|
}
|
|
|
|
func setupAvazArticlePageCrawler(crArticlePage *colly.Collector) {
|
|
crArticlePage.OnHTML("html", func(e *colly.HTMLElement) {
|
|
|
|
url := e.Request.URL.String()
|
|
|
|
title := ""
|
|
e.ForEachWithBreak("h1.title, h3.title", func(_ int, el *colly.HTMLElement) bool {
|
|
title = el.Text
|
|
return false
|
|
})
|
|
|
|
text := ""
|
|
|
|
e.ForEach("p.podtitle, div.artikal-text", func(_ int, el *colly.HTMLElement) {
|
|
text += extractJustText(el.DOM)
|
|
})
|
|
|
|
article := model.ScrapedArticle{}
|
|
|
|
trimmedText := strings.TrimSpace(text)
|
|
article.OriginalUrl = url
|
|
article.Title = title
|
|
article.Content = trimmedText
|
|
article.SourceId = model.AvazSource
|
|
slugBase := fmt.Sprintf("%d %d %s", article.SourceId, rand.Intn(1000), title)
|
|
article.Slug = slug.Make(slugBase)
|
|
|
|
AvazArticles <- article
|
|
})
|
|
|
|
crArticlePage.OnError(func(_ *colly.Response, err error) {
|
|
fmt.Println("Problem crawling!", err)
|
|
})
|
|
|
|
}
|