47 lines
856 B
Go
47 lines
856 B
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/lib/pq"
|
|
"gitlab.com/kbr4/svevijesti/internal/database"
|
|
"gitlab.com/kbr4/svevijesti/internal/scraper"
|
|
)
|
|
|
|
func main() {
|
|
store, err := database.Connect()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
defer store.Close()
|
|
go candidateChecker()
|
|
go scraper.CrawlKlix()
|
|
|
|
for article := range scraper.KlixArticles {
|
|
fmt.Println("Saving ", article.OriginalUrl)
|
|
err = database.InsertArticle(store, article)
|
|
if err, ok := err.(*pq.Error); ok {
|
|
if err.Code.Name() != "unique_violation" {
|
|
panic(err)
|
|
} else {
|
|
fmt.Println("Skipping: ", article.OriginalUrl)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func candidateChecker() {
|
|
store, err := database.Connect()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer store.Close()
|
|
|
|
for url := range scraper.KlixCandidates {
|
|
if !database.IsSaved(store, url) {
|
|
scraper.KlixApprovedSites <- url
|
|
}
|
|
}
|
|
|
|
}
|