Prva verzija - klix scraper

cmd/spider/spider.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+	"github.com/lib/pq"
+	"gitlab.com/kbr4/svevijesti/internal/database"
+	"gitlab.com/kbr4/svevijesti/internal/scraper"
+)
+
+func main() {
+	store, err := database.Connect()
+	if err != nil {
+		panic(err)
+	}
+
+	defer store.Close()
+	go candidateChecker()
+	go scraper.CrawlKlix()
+
+	for article := range scraper.KlixArticles {
+		fmt.Println("Saving ", article.OriginalUrl)
+		err = database.InsertArticle(store, article)
+		if err, ok := err.(*pq.Error); ok {
+			if err.Code.Name() != "unique_violation" {
+				panic(err)
+			} else {
+				fmt.Println("Skipping: ", article.OriginalUrl)
+			}
+		}
+	}
+}
+
+func candidateChecker() {
+	store, err := database.Connect()
+	if err != nil {
+		panic(err)
+	}
+	defer store.Close()
+
+	for url := range scraper.KlixCandidates {
+		if !database.IsSaved(store, url) {
+			scraper.KlixApprovedSites <- url
+		}
+	}
+
+}