Merge branch 'create-client-daemon' into 'master'
Create client daemon See merge request saburly/kiviscrapworker!1
This commit was merged in pull request #1.
This commit is contained in:
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
*.log
|
||||||
|
*.pid
|
||||||
|
|
||||||
|
.env
|
||||||
@@ -1,3 +1,7 @@
|
|||||||
# kiviscrapworker
|
# kiviscrapworker
|
||||||
|
|
||||||
Kivi scraping worker
|
Kivi scraping worker
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
|
Copy `example.env` to the `.env` and set desired values
|
||||||
7
example.env
Normal file
7
example.env
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
# Check default values in kiviscraplib/config/config.go
|
||||||
|
|
||||||
|
CLIENT_CONNECTIONS_COUNT = Number of connections to initiate when client worker daemon starts
|
||||||
|
CLIENT_CONNECTION_TIMEOUT = Number of seconds to wait before trying to connect to the worker server again
|
||||||
|
WORKER_SERVER_ADDRESS = Address in form of IP:PORT (127.0.0.1:1338)
|
||||||
|
REQUEST_MESSAGE_PREFIX = Prefix that worker server adds to the request message when sending to the worker client. Use double quotes if prefix has a space (eg. "URL ")
|
||||||
|
PROXY_LIST_BASE_URL = Base url where proxy list can be fetched. Proxy type(https, socks5) will be concatenated to this URL
|
||||||
36
main.go
Normal file
36
main.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/sevlyar/go-daemon"
|
||||||
|
"gitlab.com/saburly/kiviscraplib/config"
|
||||||
|
"gitlab.com/saburly/kiviscraplib/workerclient"
|
||||||
|
"log"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
config.InitClientConfig()
|
||||||
|
|
||||||
|
cntxt := &daemon.Context{
|
||||||
|
PidFileName: "sample.pid",
|
||||||
|
PidFilePerm: 0644,
|
||||||
|
LogFileName: "sample.log",
|
||||||
|
LogFilePerm: 0640,
|
||||||
|
WorkDir: "./",
|
||||||
|
Umask: 027,
|
||||||
|
Args: []string{"[kivi scraping worker client]"},
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := cntxt.Reborn()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Unable to run: ", err)
|
||||||
|
}
|
||||||
|
if d != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// TODO: Handle error
|
||||||
|
defer cntxt.Release()
|
||||||
|
|
||||||
|
go workerclient.StartClientConnections()
|
||||||
|
|
||||||
|
select {} // Wait forever
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user