senaduka/old-svevijesti
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix avaz - skip english

Browse Source
This commit is contained in:
Senad Uka
2022-02-22 21:12:12 +01:00
parent c8d361f458
commit ef6fce64d8
2 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/scraper/avaz.go
View File

@@ -37,7 +37,7 @@ func visitAvazApprovedPages(crArticlePage *colly.Collector) {
func setupAvazHomepageCrawler(crHomePage *colly.Collector, crArticlePage *colly.Collector) { func setupAvazHomepageCrawler(crHomePage *colly.Collector, crArticlePage *colly.Collector) {
articleUrlR, _ := regexp.Compile("/\\d\\d+/([a-z0-9-]+)") articleUrlR, _ := regexp.Compile("/\\d\\d+/([a-z0-9-]+)")
articleBlacklist, _ := regexp.Compile("(info-vodic|foto-data)") articleBlacklist, _ := regexp.Compile("(english)")
crHomePage.OnHTML("a", func(e *colly.HTMLElement) { crHomePage.OnHTML("a", func(e *colly.HTMLElement) {
url := e.Attr("href") url := e.Attr("href")
completeUrl := url completeUrl := url

BIN
spider
View File

Binary file not shown.