diff --git a/CHANGELOG.md b/CHANGELOG.md index d75e97038..863f7e2d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ IMPROVEMENTS: * client: Allow task's to be run as particular user [GH-950, GH-978] * client: `artifact` block now supports downloading paths relative to the task's directory [GH-944] + * discovery: Support script based health checks [GH-986] BUG FIXES: * core: Fix issue where in-place updated allocation double counted resources diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index d412582c3..87aae91e1 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -255,13 +255,13 @@ }, { "ImportPath": "github.com/hashicorp/consul/api", - "Comment": "v0.6.3-290-ge3f6c6a", - "Rev": "e3f6c6a7987ff879a1c138abcc4d14d8b65fc13f" + "Comment": "v0.6.3-363-gae32a3c", + "Rev": "ae32a3ceae9fddb431b933ed7b2a82110e41e1bf" }, { "ImportPath": "github.com/hashicorp/consul/tlsutil", - "Comment": "v0.6.3-290-ge3f6c6a", - "Rev": "e3f6c6a7987ff879a1c138abcc4d14d8b65fc13f" + "Comment": "v0.6.3-363-gae32a3c", + "Rev": "ae32a3ceae9fddb431b933ed7b2a82110e41e1bf" }, { "ImportPath": "github.com/hashicorp/errwrap", diff --git a/client/consul/sync.go b/client/consul/sync.go index fd4f022c9..19014f343 100644 --- a/client/consul/sync.go +++ b/client/consul/sync.go @@ -44,6 +44,9 @@ type ConsulConfig struct { Auth string EnableSSL bool VerifySSL bool + CAFile string + CertFile string + KeyFile string } const ( @@ -83,6 +86,20 @@ func NewConsulService(config *ConsulConfig, logger *log.Logger, allocID string) } if config.EnableSSL { cfg.Scheme = "https" + tlsCfg := consul.TLSConfig{ + Address: cfg.Address, + CAFile: config.CAFile, + CertFile: config.CertFile, + KeyFile: config.KeyFile, + InsecureSkipVerify: !config.VerifySSL, + } + tlsClientCfg, err := consul.SetupTLSConfig(&tlsCfg) + if err != nil { + return nil, fmt.Errorf("error creating tls client config for consul: %v", err) + } + cfg.HttpClient.Transport = &http.Transport{ + TLSClientConfig: tlsClientCfg, + } } if config.EnableSSL && !config.VerifySSL { cfg.HttpClient.Transport = &http.Transport{ diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 06371b724..58d20a813 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -523,7 +523,7 @@ func (e *UniversalExecutor) createCheck(check *structs.ServiceCheck, checkID str interval: check.Interval, containerID: e.consulCtx.ContainerID, logger: e.logger, - cmd: check.Cmd, + cmd: check.Command, args: check.Args, }, nil } @@ -532,7 +532,7 @@ func (e *UniversalExecutor) createCheck(check *structs.ServiceCheck, checkID str return &ExecScriptCheck{ id: checkID, interval: check.Interval, - cmd: check.Cmd, + cmd: check.Command, args: check.Args, taskDir: e.taskDir, FSIsolation: e.command.FSIsolation, diff --git a/client/driver/utils.go b/client/driver/utils.go index baab9c31a..ce1160a79 100644 --- a/client/driver/utils.go +++ b/client/driver/utils.go @@ -79,6 +79,9 @@ func consulContext(clientConfig *config.Config, containerID string) *executor.Co Auth: clientConfig.Read("consul.auth"), EnableSSL: clientConfig.ReadBoolDefault("consul.ssl", false), VerifySSL: clientConfig.ReadBoolDefault("consul.verifyssl", true), + CAFile: clientConfig.Read("consul.tls_ca_file"), + CertFile: clientConfig.Read("consul.tls_cert_file"), + KeyFile: clientConfig.Read("consul.tls_key_file"), } return &executor.ConsulContext{ ConsulConfig: &cfg, diff --git a/jobspec/parse.go b/jobspec/parse.go index acc9cc4af..d42f0836f 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -750,7 +750,7 @@ func parseChecks(service *structs.Service, checkObjs *ast.ObjectList) error { "timeout", "path", "protocol", - "cmd", + "command", "args", } if err := checkHCLKeys(co.Val, valid); err != nil { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 6082ae4e7..484cd5bbc 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1411,7 +1411,7 @@ const ( type ServiceCheck struct { Name string // Name of the check, defaults to id Type string // Type of the check - tcp, http, docker and script - Cmd string // Cmd is the command to run for script checks + Command string // Command is the command to run for script checks Args []string // Args is a list of argumes for script checks Path string // path of the health check url for http type check Protocol string // Protocol to use if check is http, defaults to http @@ -1437,7 +1437,7 @@ func (sc *ServiceCheck) Validate() error { return fmt.Errorf("service checks of http type must have a valid http path") } - if sc.Type == ServiceCheckScript && sc.Cmd == "" { + if sc.Type == ServiceCheckScript && sc.Command == "" { return fmt.Errorf("service checks of script type must have a valid script path") } @@ -1452,7 +1452,7 @@ func (sc *ServiceCheck) Hash(serviceID string) string { io.WriteString(h, serviceID) io.WriteString(h, sc.Name) io.WriteString(h, sc.Type) - io.WriteString(h, sc.Cmd) + io.WriteString(h, sc.Command) io.WriteString(h, strings.Join(sc.Args, "")) io.WriteString(h, sc.Path) io.WriteString(h, sc.Protocol) diff --git a/vendor/github.com/hashicorp/consul/api/api.go b/vendor/github.com/hashicorp/consul/api/api.go index d0e0ceeae..590b858e1 100644 --- a/vendor/github.com/hashicorp/consul/api/api.go +++ b/vendor/github.com/hashicorp/consul/api/api.go @@ -3,9 +3,11 @@ package api import ( "bytes" "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io" + "io/ioutil" "log" "net" "net/http" @@ -122,6 +124,30 @@ type Config struct { Token string } +// TLSConfig is used to generate a TLSClientConfig that's useful for talking to +// Consul using TLS. +type TLSConfig struct { + // Address is the optional address of the Consul server. The port, if any + // will be removed from here and this will be set to the ServerName of the + // resulting config. + Address string + + // CAFile is the optional path to the CA certificate used for Consul + // communication, defaults to the system bundle if not specified. + CAFile string + + // CertFile is the optional path to the certificate for Consul + // communication. If this is set then you need to also set KeyFile. + CertFile string + + // KeyFile is the optional path to the private key for Consul communication. + // If this is set then you need to also set CertFile. + KeyFile string + + // InsecureSkipVerify if set to true will disable TLS host verification. + InsecureSkipVerify bool +} + // DefaultConfig returns a default configuration for the client. By default this // will pool and reuse idle connections to Consul. If you have a long-lived // client object, this is the desired behavior and should make the most efficient @@ -194,10 +220,19 @@ func defaultConfig(transportFn func() *http.Transport) *Config { } if !doVerify { - transport := transportFn() - transport.TLSClientConfig = &tls.Config{ + tlsClientConfig, err := SetupTLSConfig(&TLSConfig{ InsecureSkipVerify: true, + }) + + // We don't expect this to fail given that we aren't + // parsing any of the input, but we panic just in case + // since this doesn't have an error return. + if err != nil { + panic(err) } + + transport := transportFn() + transport.TLSClientConfig = tlsClientConfig config.HttpClient.Transport = transport } } @@ -205,6 +240,50 @@ func defaultConfig(transportFn func() *http.Transport) *Config { return config } +// TLSConfig is used to generate a TLSClientConfig that's useful for talking to +// Consul using TLS. +func SetupTLSConfig(tlsConfig *TLSConfig) (*tls.Config, error) { + tlsClientConfig := &tls.Config{ + InsecureSkipVerify: tlsConfig.InsecureSkipVerify, + } + + if tlsConfig.Address != "" { + server := tlsConfig.Address + hasPort := strings.LastIndex(server, ":") > strings.LastIndex(server, "]") + if hasPort { + var err error + server, _, err = net.SplitHostPort(server) + if err != nil { + return nil, err + } + } + tlsClientConfig.ServerName = server + } + + if tlsConfig.CertFile != "" && tlsConfig.KeyFile != "" { + tlsCert, err := tls.LoadX509KeyPair(tlsConfig.CertFile, tlsConfig.KeyFile) + if err != nil { + return nil, err + } + tlsClientConfig.Certificates = []tls.Certificate{tlsCert} + } + + if tlsConfig.CAFile != "" { + data, err := ioutil.ReadFile(tlsConfig.CAFile) + if err != nil { + return nil, fmt.Errorf("failed to read CA file: %v", err) + } + + caPool := x509.NewCertPool() + if !caPool.AppendCertsFromPEM(data) { + return nil, fmt.Errorf("failed to parse CA certificate") + } + tlsClientConfig.RootCAs = caPool + } + + return tlsClientConfig, nil +} + // Client provides a client to the Consul API type Client struct { config Config diff --git a/website/source/docs/jobspec/servicediscovery.html.md b/website/source/docs/jobspec/servicediscovery.html.md index 7585d1c47..23680ad7e 100644 --- a/website/source/docs/jobspec/servicediscovery.html.md +++ b/website/source/docs/jobspec/servicediscovery.html.md @@ -37,6 +37,20 @@ Nomad does not currently run Consul for you. * `consul.verifyssl`: This option enables SSL verification when the transport scheme for the Consul API client is `https`. This is set to true by default. +* `consul.tls_ca_file`: The path to the CA certificate used for Consul communication. + Set accordingly to the + [ca_file](https://www.consul.io/docs/agent/options.html#ca_file) setting in + Consul. + +* `consul.tls_cert_file`: The path to the certificate for Consul communication. Set + accordingly + [cert_file](https://www.consul.io/docs/agent/options.html#cert_file) in + Consul. + +* `consul.tls_key_file`: The path to the private key for Consul communication. + Set accordingly to the + [key_file](https://www.consul.io/docs/agent/options.html#key_file) setting in + Consul. ## Service Definition Syntax @@ -61,6 +75,14 @@ group "database" { interval = "10s" timeout = "2s" } + check { + type = "script" + name = "check_table" + cmd = "/usr/local/bin/check_mysql_table_status" + args = ["--verbose"] + interval = "60s" + timeout = "5s" + } } resources { cpu = 500 @@ -97,8 +119,10 @@ group "database" { with Consul. * `check`: A check block defines a health check associated with the service. - Multiple check blocks are allowed for a service. Nomad currently supports - only the `http` and `tcp` Consul Checks. + Multiple check blocks are allowed for a service. Nomad supports the `script`, + `http` and `tcp` Consul Checks. Script checks are not supported for the qemu + driver since the Nomad client doesn't have access to the file system of a + tasks using the Qemu driver. ### Check Syntax @@ -120,8 +144,15 @@ group "database" { * `protocol`: This indicates the protocol for the http checks. Valid options are `http` and `https`. We default it to `http` +* `command`: This is the command that the Nomad client runs for doing script based + health check. + +* `args`: Additional arguments to the `command` for script based health checks. + ## Assumptions +* Consul 0.6.4 or later is needed for using the Script checks. + * Consul 0.6.0 or later is needed for using the TCP checks. * The service discovery feature in Nomad depends on operators making sure that