From c7b0d52ca44e7cdd88e466004ba05d569c571cdd Mon Sep 17 00:00:00 2001 From: Bilal Date: Thu, 7 May 2020 10:00:13 +0200 Subject: [PATCH 1/3] Create client daemon --- .gitignore | 2 ++ main.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 .gitignore create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58bce9c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.log +*.pid \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..c7a948e --- /dev/null +++ b/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "github.com/sevlyar/go-daemon" + "gitlab.com/saburly/kiviscraplib/workerclient" + "log" +) + +func main() { + cntxt := &daemon.Context{ + PidFileName: "sample.pid", + PidFilePerm: 0644, + LogFileName: "sample.log", + LogFilePerm: 0640, + WorkDir: "./", + Umask: 027, + Args: []string{"[kivi scraping worker client]"}, + } + + d, err := cntxt.Reborn() + if err != nil { + log.Fatal("Unable to run: ", err) + } + if d != nil { + return + } + // TODO: Handle error + defer cntxt.Release() + + go workerclient.StartClientConnections() + + select {} // Wait forever +} From b4153c051c53edc3363dd354c66a0de5468225ee Mon Sep 17 00:00:00 2001 From: Bilal Date: Fri, 8 May 2020 07:47:09 +0200 Subject: [PATCH 2/3] Load config from ENV file --- .gitignore | 4 +++- README.md | 6 +++++- example.env | 7 +++++++ main.go | 3 +++ 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 example.env diff --git a/.gitignore b/.gitignore index 58bce9c..11b4d34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.log -*.pid \ No newline at end of file +*.pid + +.env \ No newline at end of file diff --git a/README.md b/README.md index 9de5c24..83a4e52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ # kiviscrapworker -Kivi scraping worker \ No newline at end of file +Kivi scraping worker + +## Setup + +Copy `example.env` to the `.env` and set desired values \ No newline at end of file diff --git a/example.env b/example.env new file mode 100644 index 0000000..530f178 --- /dev/null +++ b/example.env @@ -0,0 +1,7 @@ +# Check default values in kiviscraplib/config/config.go + +CLIENT_CONNECTIONS_COUNT = Number of connections to initiate when client worker daemon starts +CLIENT_CONNECTION_TIMEOUT = Number of seconds to wait before trying to connect to the worker server again +WORKER_SERVER_ADDRESS = Address in form of IP:PORT (127.0.0.1:1338) +REQUEST_MESSAGE_PREFIX = Prefix that worker server adds to the request message when sending to the worker client. Use double quotes if prefix has a space (eg. "URL ") +PROXY_LIST_BASE_URL = Base url where proxy list can be fetched. Proxy type(https, socks5) will be concatenated to this URL \ No newline at end of file diff --git a/main.go b/main.go index c7a948e..bc9f0a3 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,14 @@ package main import ( "github.com/sevlyar/go-daemon" + "gitlab.com/saburly/kiviscraplib/config" "gitlab.com/saburly/kiviscraplib/workerclient" "log" ) func main() { + config.InitConfig() + cntxt := &daemon.Context{ PidFileName: "sample.pid", PidFilePerm: 0644, From df528e50136a93e84d3878f4fd6111bd55787527 Mon Sep 17 00:00:00 2001 From: Bilal Date: Fri, 8 May 2020 10:45:00 +0200 Subject: [PATCH 3/3] Load client config when starting client daemon --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index bc9f0a3..253fd48 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,7 @@ import ( ) func main() { - config.InitConfig() + config.InitClientConfig() cntxt := &daemon.Context{ PidFileName: "sample.pid",