Fix avaz - skip english

This commit is contained in:
Senad Uka
2022-02-22 21:12:12 +01:00
parent c8d361f458
commit ef6fce64d8
2 changed files with 1 additions and 1 deletions

View File

@@ -37,7 +37,7 @@ func visitAvazApprovedPages(crArticlePage *colly.Collector) {
func setupAvazHomepageCrawler(crHomePage *colly.Collector, crArticlePage *colly.Collector) {
articleUrlR, _ := regexp.Compile("/\\d\\d+/([a-z0-9-]+)")
articleBlacklist, _ := regexp.Compile("(info-vodic|foto-data)")
articleBlacklist, _ := regexp.Compile("(english)")
crHomePage.OnHTML("a", func(e *colly.HTMLElement) {
url := e.Attr("href")
completeUrl := url

BIN
spider

Binary file not shown.