From 3c149e6fd698a33ede81b56668cb87ce9da07631 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Sat, 26 Mar 2016 13:02:30 -0700 Subject: [PATCH 1/7] Added docs for script checks --- CHANGELOG.md | 1 + .../docs/jobspec/servicediscovery.html.md | 21 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d75e97038..863f7e2d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ IMPROVEMENTS: * client: Allow task's to be run as particular user [GH-950, GH-978] * client: `artifact` block now supports downloading paths relative to the task's directory [GH-944] + * discovery: Support script based health checks [GH-986] BUG FIXES: * core: Fix issue where in-place updated allocation double counted resources diff --git a/website/source/docs/jobspec/servicediscovery.html.md b/website/source/docs/jobspec/servicediscovery.html.md index 7585d1c47..486b7ca36 100644 --- a/website/source/docs/jobspec/servicediscovery.html.md +++ b/website/source/docs/jobspec/servicediscovery.html.md @@ -61,6 +61,14 @@ group "database" { interval = "10s" timeout = "2s" } + check { + type = "script" + name = "check_table" + cmd = "/usr/local/bin/check_mysql_table_status" + agrs = ["--verbose"] + interval = "60s" + timeout = "5s" + } } resources { cpu = 500 @@ -97,8 +105,10 @@ group "database" { with Consul. * `check`: A check block defines a health check associated with the service. - Multiple check blocks are allowed for a service. Nomad currently supports - only the `http` and `tcp` Consul Checks. + Multiple check blocks are allowed for a service. Nomad supports the `script`, + `http` and `tcp` Consul Checks. Script checks are not supported for the qemu + driver since the Nomad client doesn't have access to the file system of a + tasks using the Qemu driver. ### Check Syntax @@ -120,9 +130,14 @@ group "database" { * `protocol`: This indicates the protocol for the http checks. Valid options are `http` and `https`. We default it to `http` +* `cmd`: This is the command that the Nomad client runs for doing script based + health check. + +* `args`: Additional arguments to the `cmd` for script based health checks. + ## Assumptions -* Consul 0.6.0 or later is needed for using the TCP checks. +* Consul 0.6.4 or later is needed for using the TCP checks. * The service discovery feature in Nomad depends on operators making sure that the Nomad client can reach the Consul agent. From 5bec27a9a46731dda6c6a581deeb40d2561f6062 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Sun, 27 Mar 2016 23:09:31 -0700 Subject: [PATCH 2/7] Using tls cert and key files while connecting to consul over https --- Godeps/Godeps.json | 8 +- client/consul/sync.go | 17 ++++ client/driver/utils.go | 3 + vendor/github.com/hashicorp/consul/api/api.go | 83 ++++++++++++++++++- 4 files changed, 105 insertions(+), 6 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index d412582c3..87aae91e1 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -255,13 +255,13 @@ }, { "ImportPath": "github.com/hashicorp/consul/api", - "Comment": "v0.6.3-290-ge3f6c6a", - "Rev": "e3f6c6a7987ff879a1c138abcc4d14d8b65fc13f" + "Comment": "v0.6.3-363-gae32a3c", + "Rev": "ae32a3ceae9fddb431b933ed7b2a82110e41e1bf" }, { "ImportPath": "github.com/hashicorp/consul/tlsutil", - "Comment": "v0.6.3-290-ge3f6c6a", - "Rev": "e3f6c6a7987ff879a1c138abcc4d14d8b65fc13f" + "Comment": "v0.6.3-363-gae32a3c", + "Rev": "ae32a3ceae9fddb431b933ed7b2a82110e41e1bf" }, { "ImportPath": "github.com/hashicorp/errwrap", diff --git a/client/consul/sync.go b/client/consul/sync.go index fd4f022c9..19014f343 100644 --- a/client/consul/sync.go +++ b/client/consul/sync.go @@ -44,6 +44,9 @@ type ConsulConfig struct { Auth string EnableSSL bool VerifySSL bool + CAFile string + CertFile string + KeyFile string } const ( @@ -83,6 +86,20 @@ func NewConsulService(config *ConsulConfig, logger *log.Logger, allocID string) } if config.EnableSSL { cfg.Scheme = "https" + tlsCfg := consul.TLSConfig{ + Address: cfg.Address, + CAFile: config.CAFile, + CertFile: config.CertFile, + KeyFile: config.KeyFile, + InsecureSkipVerify: !config.VerifySSL, + } + tlsClientCfg, err := consul.SetupTLSConfig(&tlsCfg) + if err != nil { + return nil, fmt.Errorf("error creating tls client config for consul: %v", err) + } + cfg.HttpClient.Transport = &http.Transport{ + TLSClientConfig: tlsClientCfg, + } } if config.EnableSSL && !config.VerifySSL { cfg.HttpClient.Transport = &http.Transport{ diff --git a/client/driver/utils.go b/client/driver/utils.go index baab9c31a..2960b0929 100644 --- a/client/driver/utils.go +++ b/client/driver/utils.go @@ -79,6 +79,9 @@ func consulContext(clientConfig *config.Config, containerID string) *executor.Co Auth: clientConfig.Read("consul.auth"), EnableSSL: clientConfig.ReadBoolDefault("consul.ssl", false), VerifySSL: clientConfig.ReadBoolDefault("consul.verifyssl", true), + CAFile: clientConfig.Read("consul.cafile"), + CertFile: clientConfig.Read("consul.certfile"), + KeyFile: clientConfig.Read("consul.keyfile"), } return &executor.ConsulContext{ ConsulConfig: &cfg, diff --git a/vendor/github.com/hashicorp/consul/api/api.go b/vendor/github.com/hashicorp/consul/api/api.go index d0e0ceeae..590b858e1 100644 --- a/vendor/github.com/hashicorp/consul/api/api.go +++ b/vendor/github.com/hashicorp/consul/api/api.go @@ -3,9 +3,11 @@ package api import ( "bytes" "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io" + "io/ioutil" "log" "net" "net/http" @@ -122,6 +124,30 @@ type Config struct { Token string } +// TLSConfig is used to generate a TLSClientConfig that's useful for talking to +// Consul using TLS. +type TLSConfig struct { + // Address is the optional address of the Consul server. The port, if any + // will be removed from here and this will be set to the ServerName of the + // resulting config. + Address string + + // CAFile is the optional path to the CA certificate used for Consul + // communication, defaults to the system bundle if not specified. + CAFile string + + // CertFile is the optional path to the certificate for Consul + // communication. If this is set then you need to also set KeyFile. + CertFile string + + // KeyFile is the optional path to the private key for Consul communication. + // If this is set then you need to also set CertFile. + KeyFile string + + // InsecureSkipVerify if set to true will disable TLS host verification. + InsecureSkipVerify bool +} + // DefaultConfig returns a default configuration for the client. By default this // will pool and reuse idle connections to Consul. If you have a long-lived // client object, this is the desired behavior and should make the most efficient @@ -194,10 +220,19 @@ func defaultConfig(transportFn func() *http.Transport) *Config { } if !doVerify { - transport := transportFn() - transport.TLSClientConfig = &tls.Config{ + tlsClientConfig, err := SetupTLSConfig(&TLSConfig{ InsecureSkipVerify: true, + }) + + // We don't expect this to fail given that we aren't + // parsing any of the input, but we panic just in case + // since this doesn't have an error return. + if err != nil { + panic(err) } + + transport := transportFn() + transport.TLSClientConfig = tlsClientConfig config.HttpClient.Transport = transport } } @@ -205,6 +240,50 @@ func defaultConfig(transportFn func() *http.Transport) *Config { return config } +// TLSConfig is used to generate a TLSClientConfig that's useful for talking to +// Consul using TLS. +func SetupTLSConfig(tlsConfig *TLSConfig) (*tls.Config, error) { + tlsClientConfig := &tls.Config{ + InsecureSkipVerify: tlsConfig.InsecureSkipVerify, + } + + if tlsConfig.Address != "" { + server := tlsConfig.Address + hasPort := strings.LastIndex(server, ":") > strings.LastIndex(server, "]") + if hasPort { + var err error + server, _, err = net.SplitHostPort(server) + if err != nil { + return nil, err + } + } + tlsClientConfig.ServerName = server + } + + if tlsConfig.CertFile != "" && tlsConfig.KeyFile != "" { + tlsCert, err := tls.LoadX509KeyPair(tlsConfig.CertFile, tlsConfig.KeyFile) + if err != nil { + return nil, err + } + tlsClientConfig.Certificates = []tls.Certificate{tlsCert} + } + + if tlsConfig.CAFile != "" { + data, err := ioutil.ReadFile(tlsConfig.CAFile) + if err != nil { + return nil, fmt.Errorf("failed to read CA file: %v", err) + } + + caPool := x509.NewCertPool() + if !caPool.AppendCertsFromPEM(data) { + return nil, fmt.Errorf("failed to parse CA certificate") + } + tlsClientConfig.RootCAs = caPool + } + + return tlsClientConfig, nil +} + // Client provides a client to the Consul API type Client struct { config Config From 8499f091e3f5cdb08a6564bc2da6be0bcc7d4310 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Sun, 27 Mar 2016 23:21:50 -0700 Subject: [PATCH 3/7] Added docs --- client/driver/utils.go | 6 +++--- .../source/docs/jobspec/servicediscovery.html.md | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/client/driver/utils.go b/client/driver/utils.go index 2960b0929..ce1160a79 100644 --- a/client/driver/utils.go +++ b/client/driver/utils.go @@ -79,9 +79,9 @@ func consulContext(clientConfig *config.Config, containerID string) *executor.Co Auth: clientConfig.Read("consul.auth"), EnableSSL: clientConfig.ReadBoolDefault("consul.ssl", false), VerifySSL: clientConfig.ReadBoolDefault("consul.verifyssl", true), - CAFile: clientConfig.Read("consul.cafile"), - CertFile: clientConfig.Read("consul.certfile"), - KeyFile: clientConfig.Read("consul.keyfile"), + CAFile: clientConfig.Read("consul.tls_ca_file"), + CertFile: clientConfig.Read("consul.tls_cert_file"), + KeyFile: clientConfig.Read("consul.tls_key_file"), } return &executor.ConsulContext{ ConsulConfig: &cfg, diff --git a/website/source/docs/jobspec/servicediscovery.html.md b/website/source/docs/jobspec/servicediscovery.html.md index 7585d1c47..46471f4c7 100644 --- a/website/source/docs/jobspec/servicediscovery.html.md +++ b/website/source/docs/jobspec/servicediscovery.html.md @@ -37,6 +37,20 @@ Nomad does not currently run Consul for you. * `consul.verifyssl`: This option enables SSL verification when the transport scheme for the Consul API client is `https`. This is set to true by default. +* `consul.tls_ca_file`: The path to the CA certificate used for Consul communication. + Set accordingly to the + [ca_file](https://www.consul.io/docs/agent/options.html#ca_file) setting in + Consul. + +* `consul.tls_cert_file`: The path to the certificate for Consul communication. Set + accordingly + [cert_file](https://www.consul.io/docs/agent/options.html#cert_file) in + Consul. + +* `consul.tls_key_file`: The path to the private key for Consul communication. + Set accordingly to the + [key_file](https://www.consul.io/docs/agent/options.html#key_file) setting in + Consul.j ## Service Definition Syntax From 77a244b80d7bacb8dc8fd238240e2c70e9513fd9 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 28 Mar 2016 10:06:44 -0700 Subject: [PATCH 4/7] Corrected docs --- website/source/docs/jobspec/servicediscovery.html.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/website/source/docs/jobspec/servicediscovery.html.md b/website/source/docs/jobspec/servicediscovery.html.md index 486b7ca36..ff12a4557 100644 --- a/website/source/docs/jobspec/servicediscovery.html.md +++ b/website/source/docs/jobspec/servicediscovery.html.md @@ -65,7 +65,7 @@ group "database" { type = "script" name = "check_table" cmd = "/usr/local/bin/check_mysql_table_status" - agrs = ["--verbose"] + args = ["--verbose"] interval = "60s" timeout = "5s" } @@ -137,7 +137,9 @@ group "database" { ## Assumptions -* Consul 0.6.4 or later is needed for using the TCP checks. +* Consul 0.6.4 or later is needed for using the Script checks. + +* Consul 0.6.0 or later is needed for using the TCP checks. * The service discovery feature in Nomad depends on operators making sure that the Nomad client can reach the Consul agent. From b49335b18a59619258885638e8f9a78c6c038410 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 28 Mar 2016 14:05:12 -0700 Subject: [PATCH 5/7] Renamed checks cmd to command to be consistent with exec driver config --- client/driver/executor/executor.go | 4 ++-- jobspec/parse.go | 2 +- nomad/structs/structs.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 06371b724..58d20a813 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -523,7 +523,7 @@ func (e *UniversalExecutor) createCheck(check *structs.ServiceCheck, checkID str interval: check.Interval, containerID: e.consulCtx.ContainerID, logger: e.logger, - cmd: check.Cmd, + cmd: check.Command, args: check.Args, }, nil } @@ -532,7 +532,7 @@ func (e *UniversalExecutor) createCheck(check *structs.ServiceCheck, checkID str return &ExecScriptCheck{ id: checkID, interval: check.Interval, - cmd: check.Cmd, + cmd: check.Command, args: check.Args, taskDir: e.taskDir, FSIsolation: e.command.FSIsolation, diff --git a/jobspec/parse.go b/jobspec/parse.go index acc9cc4af..d42f0836f 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -750,7 +750,7 @@ func parseChecks(service *structs.Service, checkObjs *ast.ObjectList) error { "timeout", "path", "protocol", - "cmd", + "command", "args", } if err := checkHCLKeys(co.Val, valid); err != nil { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 6082ae4e7..484cd5bbc 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1411,7 +1411,7 @@ const ( type ServiceCheck struct { Name string // Name of the check, defaults to id Type string // Type of the check - tcp, http, docker and script - Cmd string // Cmd is the command to run for script checks + Command string // Command is the command to run for script checks Args []string // Args is a list of argumes for script checks Path string // path of the health check url for http type check Protocol string // Protocol to use if check is http, defaults to http @@ -1437,7 +1437,7 @@ func (sc *ServiceCheck) Validate() error { return fmt.Errorf("service checks of http type must have a valid http path") } - if sc.Type == ServiceCheckScript && sc.Cmd == "" { + if sc.Type == ServiceCheckScript && sc.Command == "" { return fmt.Errorf("service checks of script type must have a valid script path") } @@ -1452,7 +1452,7 @@ func (sc *ServiceCheck) Hash(serviceID string) string { io.WriteString(h, serviceID) io.WriteString(h, sc.Name) io.WriteString(h, sc.Type) - io.WriteString(h, sc.Cmd) + io.WriteString(h, sc.Command) io.WriteString(h, strings.Join(sc.Args, "")) io.WriteString(h, sc.Path) io.WriteString(h, sc.Protocol) From e52cb8e65c5e90a61eb2a0a31ca98da33202520d Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 28 Mar 2016 14:25:15 -0700 Subject: [PATCH 6/7] Updated docs --- website/source/docs/jobspec/servicediscovery.html.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/source/docs/jobspec/servicediscovery.html.md b/website/source/docs/jobspec/servicediscovery.html.md index ff12a4557..38067cb88 100644 --- a/website/source/docs/jobspec/servicediscovery.html.md +++ b/website/source/docs/jobspec/servicediscovery.html.md @@ -130,10 +130,10 @@ group "database" { * `protocol`: This indicates the protocol for the http checks. Valid options are `http` and `https`. We default it to `http` -* `cmd`: This is the command that the Nomad client runs for doing script based +* `command`: This is the command that the Nomad client runs for doing script based health check. -* `args`: Additional arguments to the `cmd` for script based health checks. +* `args`: Additional arguments to the `command` for script based health checks. ## Assumptions From 310cf4490b0ab02e3788208ff734adf602e2b5d5 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 28 Mar 2016 14:26:31 -0700 Subject: [PATCH 7/7] Removing extra chars --- website/source/docs/jobspec/servicediscovery.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/jobspec/servicediscovery.html.md b/website/source/docs/jobspec/servicediscovery.html.md index 46471f4c7..0a9d21b39 100644 --- a/website/source/docs/jobspec/servicediscovery.html.md +++ b/website/source/docs/jobspec/servicediscovery.html.md @@ -50,7 +50,7 @@ Nomad does not currently run Consul for you. * `consul.tls_key_file`: The path to the private key for Consul communication. Set accordingly to the [key_file](https://www.consul.io/docs/agent/options.html#key_file) setting in - Consul.j + Consul. ## Service Definition Syntax