Added avaz / fixed install script

This commit is contained in:
Senad Uka
2022-02-22 21:06:27 +01:00
parent 2db4b0b2e2
commit c8d361f458
6 changed files with 124 additions and 0 deletions

View File

@@ -19,6 +19,7 @@ func main() {
go scraper.CrawlSrpskainfo()
go scraper.CrawlKlix()
go scraper.CrawlBljesak()
go scraper.CrawlAvaz()
article := model.ScrapedArticle{}
@@ -36,6 +37,10 @@ func main() {
if article.Title == model.Terminator {
scraper.BljesakArticles = nil
}
case article = <-scraper.AvazArticles:
if article.Title == model.Terminator {
scraper.AvazArticles = nil
}
}
@@ -55,6 +60,7 @@ func main() {
if scraper.KlixArticles == nil &&
scraper.SrpskainfoArticles == nil &&
scraper.AvazArticles == nil &&
scraper.BljesakCandidates == nil {
break
}
@@ -97,10 +103,20 @@ func candidateChecker() {
}
}
case url := <-scraper.AvazCandidates:
if url == model.Terminator {
scraper.AvazCandidates = nil
} else {
if !database.IsSaved(store, url) {
scraper.AvazApprovedSites <- url
}
}
}
if scraper.KlixCandidates == nil &&
scraper.SrpskainfoCandidates == nil &&
scraper.AvazCandidates == nil &&
scraper.BljesakCandidates == nil {
break
}