diff --git a/.gitignore b/.gitignore index 58bce9c..11b4d34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.log -*.pid \ No newline at end of file +*.pid + +.env \ No newline at end of file diff --git a/README.md b/README.md index 9de5c24..83a4e52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ # kiviscrapworker -Kivi scraping worker \ No newline at end of file +Kivi scraping worker + +## Setup + +Copy `example.env` to the `.env` and set desired values \ No newline at end of file diff --git a/example.env b/example.env new file mode 100644 index 0000000..530f178 --- /dev/null +++ b/example.env @@ -0,0 +1,7 @@ +# Check default values in kiviscraplib/config/config.go + +CLIENT_CONNECTIONS_COUNT = Number of connections to initiate when client worker daemon starts +CLIENT_CONNECTION_TIMEOUT = Number of seconds to wait before trying to connect to the worker server again +WORKER_SERVER_ADDRESS = Address in form of IP:PORT (127.0.0.1:1338) +REQUEST_MESSAGE_PREFIX = Prefix that worker server adds to the request message when sending to the worker client. Use double quotes if prefix has a space (eg. "URL ") +PROXY_LIST_BASE_URL = Base url where proxy list can be fetched. Proxy type(https, socks5) will be concatenated to this URL \ No newline at end of file diff --git a/main.go b/main.go index c7a948e..bc9f0a3 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,14 @@ package main import ( "github.com/sevlyar/go-daemon" + "gitlab.com/saburly/kiviscraplib/config" "gitlab.com/saburly/kiviscraplib/workerclient" "log" ) func main() { + config.InitConfig() + cntxt := &daemon.Context{ PidFileName: "sample.pid", PidFilePerm: 0644,