diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11b4d34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.log +*.pid + +.env \ No newline at end of file diff --git a/README.md b/README.md index 9de5c24..83a4e52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ # kiviscrapworker -Kivi scraping worker \ No newline at end of file +Kivi scraping worker + +## Setup + +Copy `example.env` to the `.env` and set desired values \ No newline at end of file diff --git a/example.env b/example.env new file mode 100644 index 0000000..530f178 --- /dev/null +++ b/example.env @@ -0,0 +1,7 @@ +# Check default values in kiviscraplib/config/config.go + +CLIENT_CONNECTIONS_COUNT = Number of connections to initiate when client worker daemon starts +CLIENT_CONNECTION_TIMEOUT = Number of seconds to wait before trying to connect to the worker server again +WORKER_SERVER_ADDRESS = Address in form of IP:PORT (127.0.0.1:1338) +REQUEST_MESSAGE_PREFIX = Prefix that worker server adds to the request message when sending to the worker client. Use double quotes if prefix has a space (eg. "URL ") +PROXY_LIST_BASE_URL = Base url where proxy list can be fetched. Proxy type(https, socks5) will be concatenated to this URL \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..253fd48 --- /dev/null +++ b/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "github.com/sevlyar/go-daemon" + "gitlab.com/saburly/kiviscraplib/config" + "gitlab.com/saburly/kiviscraplib/workerclient" + "log" +) + +func main() { + config.InitClientConfig() + + cntxt := &daemon.Context{ + PidFileName: "sample.pid", + PidFilePerm: 0644, + LogFileName: "sample.log", + LogFilePerm: 0640, + WorkDir: "./", + Umask: 027, + Args: []string{"[kivi scraping worker client]"}, + } + + d, err := cntxt.Reborn() + if err != nil { + log.Fatal("Unable to run: ", err) + } + if d != nil { + return + } + // TODO: Handle error + defer cntxt.Release() + + go workerclient.StartClientConnections() + + select {} // Wait forever +}