vendor update

This commit is contained in:
Alex Dadgar
2017-01-13 16:22:27 -08:00
parent bcd2a448c5
commit a09fc0ff60
54 changed files with 5807 additions and 3527 deletions

View File

@@ -2,11 +2,13 @@ package child
import (
"errors"
"fmt"
"io"
"log"
"math/rand"
"os"
"os/exec"
"strings"
"sync"
"syscall"
"time"
@@ -15,14 +17,14 @@ import (
var (
// ErrMissingCommand is the error returned when no command is specified
// to run.
ErrMissingCommand error = errors.New("missing command")
ErrMissingCommand = errors.New("missing command")
// ExitCodeOK is the default OK exit code.
ExitCodeOK int = 0
ExitCodeOK = 0
// ExitCodeError is the default error code returned when the child exits with
// an error without a more specific code.
ExitCodeError int = 127
ExitCodeError = 127
)
// Child is a wrapper around a child process which can be used to send signals
@@ -34,6 +36,9 @@ type Child struct {
stdout, stderr io.Writer
command string
args []string
env []string
timeout time.Duration
reloadSignal os.Signal
@@ -69,6 +74,16 @@ type NewInput struct {
Command string
Args []string
// Timeout is the maximum amount of time to allow the command to execute. If
// set to 0, the command is permitted to run infinitely.
Timeout time.Duration
// Env represents the condition of the child processes' environment
// variables. Only these environment variables will be given to the child, so
// it is the responsibility of the caller to include the parent processes
// environment, if required. This should be in the key=value format.
Env []string
// ReloadSignal is the signal to send to reload this process. This value may
// be nil.
ReloadSignal os.Signal
@@ -107,6 +122,8 @@ func New(i *NewInput) (*Child, error) {
stderr: i.Stderr,
command: i.Command,
args: i.Args,
env: i.Env,
timeout: i.Timeout,
reloadSignal: i.ReloadSignal,
killSignal: i.KillSignal,
killTimeout: i.KillTimeout,
@@ -134,13 +151,19 @@ func (c *Child) Pid() int {
return c.pid()
}
// Command returns the human-formatted command with arguments.
func (c *Child) Command() string {
list := append([]string{c.command}, c.args...)
return strings.Join(list, " ")
}
// Start starts and begins execution of the child process. A buffered channel
// is returned which is where the command's exit code will be returned upon
// exit. Any errors that occur prior to starting the command will be returned
// as the second error argument, but any errors returned by the command after
// execution will be returned as a non-zero value over the exit code channel.
func (c *Child) Start() error {
log.Printf("[INFO] (child) spawning %q %q", c.command, c.args)
log.Printf("[INFO] (child) spawning: %s", c.Command())
c.Lock()
defer c.Unlock()
return c.start()
@@ -170,16 +193,16 @@ func (c *Child) Reload() error {
c.kill()
return c.start()
} else {
log.Printf("[INFO] (child) reloading process")
// We only need a read lock here because neither the process nor the exit
// channel are changing.
c.RLock()
defer c.RUnlock()
return c.reload()
}
log.Printf("[INFO] (child) reloading process")
// We only need a read lock here because neither the process nor the exit
// channel are changing.
c.RLock()
defer c.RUnlock()
return c.reload()
}
// Kill sends the kill signal to the child process and waits for successful
@@ -223,6 +246,7 @@ func (c *Child) start() error {
cmd.Stdin = c.stdin
cmd.Stdout = c.stdout
cmd.Stderr = c.stderr
cmd.Env = c.env
if err := cmd.Start(); err != nil {
return err
}
@@ -257,10 +281,47 @@ func (c *Child) start() error {
case <-c.stopCh:
case exitCh <- code:
}
}()
c.exitCh = exitCh
// If a timeout was given, start the timer to wait for the child to exit
if c.timeout != 0 {
select {
case code := <-exitCh:
if code != 0 {
return fmt.Errorf(
"command exited with a non-zero exit status:\n"+
"\n"+
" %s\n"+
"\n"+
"This is assumed to be a failure. Please ensure the command\n"+
"exits with a zero exit status.",
c.Command(),
)
}
case <-time.After(c.timeout):
// Force-kill the process
c.stopLock.Lock()
defer c.stopLock.Unlock()
if c.cmd != nil && c.cmd.Process != nil {
c.cmd.Process.Kill()
}
return fmt.Errorf(
"command did not exit within %q:\n"+
"\n"+
" %s\n"+
"\n"+
"Commands must exit in a timely manner in order for processing to\n"+
"continue. Consider using a process supervisor or utilizing the\n"+
"built-in exec mode instead.",
c.timeout,
c.Command(),
)
}
}
return nil
}

View File

@@ -0,0 +1,142 @@
package config
import (
"errors"
"fmt"
"strings"
)
var (
// ErrAuthStringEmpty is the error returned with authentication is provided,
// but empty.
ErrAuthStringEmpty = errors.New("auth: cannot be empty")
)
// AuthConfig is the HTTP basic authentication data.
type AuthConfig struct {
Enabled *bool `mapstructure:"enabled"`
Username *string `mapstructure:"username"`
Password *string `mapstructure:"password"`
}
// DefaultAuthConfig is the default configuration.
func DefaultAuthConfig() *AuthConfig {
return &AuthConfig{}
}
// ParseAuthConfig parses the auth into username:password.
func ParseAuthConfig(s string) (*AuthConfig, error) {
if s == "" {
return nil, ErrAuthStringEmpty
}
var a AuthConfig
if strings.Contains(s, ":") {
split := strings.SplitN(s, ":", 2)
a.Username = String(split[0])
a.Password = String(split[1])
} else {
a.Username = String(s)
}
return &a, nil
}
// Copy returns a deep copy of this configuration.
func (c *AuthConfig) Copy() *AuthConfig {
if c == nil {
return nil
}
var o AuthConfig
o.Enabled = c.Enabled
o.Username = c.Username
o.Password = c.Password
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *AuthConfig) Merge(o *AuthConfig) *AuthConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Enabled != nil {
r.Enabled = o.Enabled
}
if o.Username != nil {
r.Username = o.Username
}
if o.Password != nil {
r.Password = o.Password
}
return r
}
// Finalize ensures there no nil pointers.
func (c *AuthConfig) Finalize() {
if c.Enabled == nil {
c.Enabled = Bool(false ||
StringPresent(c.Username) ||
StringPresent(c.Password))
}
if c.Username == nil {
c.Username = String("")
}
if c.Password == nil {
c.Password = String("")
}
if c.Enabled == nil {
c.Enabled = Bool(*c.Username != "" || *c.Password != "")
}
}
// GoString defines the printable version of this struct.
func (c *AuthConfig) GoString() string {
if c == nil {
return "(*AuthConfig)(nil)"
}
return fmt.Sprintf("&AuthConfig{"+
"Enabled:%s, "+
"Username:%s, "+
"Password:%s"+
"}",
BoolGoString(c.Enabled),
StringGoString(c.Username),
StringGoString(c.Password),
)
}
// String is the string representation of this authentication. If authentication
// is not enabled, this returns the empty string. The username and password will
// be separated by a colon.
func (c *AuthConfig) String() string {
if !BoolVal(c.Enabled) {
return ""
}
if c.Password != nil {
return fmt.Sprintf("%s:%s", StringVal(c.Username), StringVal(c.Password))
}
return StringVal(c.Username)
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,150 @@
package config
import "fmt"
// ConsulConfig contains the configurations options for connecting to a
// Consul cluster.
type ConsulConfig struct {
// Address is the address of the Consul server. It may be an IP or FQDN.
Address *string
// Auth is the HTTP basic authentication for communicating with Consul.
Auth *AuthConfig `mapstructure:"auth"`
// Retry is the configuration for specifying how to behave on failure.
Retry *RetryConfig `mapstructure:"retry"`
// SSL indicates we should use a secure connection while talking to
// Consul. This requires Consul to be configured to serve HTTPS.
SSL *SSLConfig `mapstructure:"ssl"`
// Token is the token to communicate with Consul securely.
Token *string
}
// DefaultConsulConfig returns a configuration that is populated with the
// default values.
func DefaultConsulConfig() *ConsulConfig {
return &ConsulConfig{
Address: stringFromEnv("CONSUL_HTTP_ADDR"),
Auth: DefaultAuthConfig(),
Retry: DefaultRetryConfig(),
SSL: DefaultSSLConfig(),
Token: stringFromEnv("CONSUL_TOKEN", "CONSUL_HTTP_TOKEN"),
}
}
// Copy returns a deep copy of this configuration.
func (c *ConsulConfig) Copy() *ConsulConfig {
if c == nil {
return nil
}
var o ConsulConfig
o.Address = c.Address
if c.Auth != nil {
o.Auth = c.Auth.Copy()
}
if c.Retry != nil {
o.Retry = c.Retry.Copy()
}
if c.SSL != nil {
o.SSL = c.SSL.Copy()
}
o.Token = c.Token
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *ConsulConfig) Merge(o *ConsulConfig) *ConsulConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Address != nil {
r.Address = o.Address
}
if o.Auth != nil {
r.Auth = r.Auth.Merge(o.Auth)
}
if o.Retry != nil {
r.Retry = r.Retry.Merge(o.Retry)
}
if o.SSL != nil {
r.SSL = r.SSL.Merge(o.SSL)
}
if o.Token != nil {
r.Token = o.Token
}
return r
}
// Finalize ensures there no nil pointers.
func (c *ConsulConfig) Finalize() {
if c.Address == nil {
c.Address = String("")
}
if c.Auth == nil {
c.Auth = DefaultAuthConfig()
}
c.Auth.Finalize()
if c.Retry == nil {
c.Retry = DefaultRetryConfig()
}
c.Retry.Finalize()
if c.SSL == nil {
c.SSL = DefaultSSLConfig()
}
c.SSL.Finalize()
if c.Token == nil {
c.Token = String("")
}
}
// GoString defines the printable version of this struct.
func (c *ConsulConfig) GoString() string {
if c == nil {
return "(*ConsulConfig)(nil)"
}
return fmt.Sprintf("&ConsulConfig{"+
"Address:%s, "+
"Auth:%#v, "+
"Retry:%#v, "+
"SSL:%#v, "+
"Token:%t"+
"}",
StringGoString(c.Address),
c.Auth,
c.Retry,
c.SSL,
StringPresent(c.Token),
)
}

View File

@@ -0,0 +1,197 @@
package config
import (
"fmt"
"os"
"time"
"github.com/hashicorp/consul-template/signals"
)
// Bool returns a pointer to the given bool.
func Bool(b bool) *bool {
return &b
}
// BoolVal returns the value of the boolean at the pointer, or false if the
// pointer is nil.
func BoolVal(b *bool) bool {
if b == nil {
return false
}
return *b
}
// BoolGoString returns the value of the boolean for printing in a string.
func BoolGoString(b *bool) string {
if b == nil {
return "(*bool)(nil)"
}
return fmt.Sprintf("%t", *b)
}
// BoolPresent returns a boolean indiciating if the pointer is nil, or if the
// pointer is pointing to the zero value..
func BoolPresent(b *bool) bool {
if b == nil {
return false
}
return true
}
// FileMode returns a pointer to the given os.FileMode.
func FileMode(o os.FileMode) *os.FileMode {
return &o
}
// FileModeVal returns the value of the os.FileMode at the pointer, or 0 if the
// pointer is nil.
func FileModeVal(o *os.FileMode) os.FileMode {
if o == nil {
return 0
}
return *o
}
// FileModeGoString returns the value of the os.FileMode for printing in a
// string.
func FileModeGoString(o *os.FileMode) string {
if o == nil {
return "(*os.FileMode)(nil)"
}
return fmt.Sprintf("%q", *o)
}
// FileModePresent returns a boolean indiciating if the pointer is nil, or if
// the pointer is pointing to the zero value.
func FileModePresent(o *os.FileMode) bool {
if o == nil {
return false
}
return *o != 0
}
// Int returns a pointer to the given int.
func Int(i int) *int {
return &i
}
// IntVal returns the value of the int at the pointer, or 0 if the pointer is
// nil.
func IntVal(i *int) int {
if i == nil {
return 0
}
return *i
}
// IntGoString returns the value of the int for printing in a string.
func IntGoString(i *int) string {
if i == nil {
return "(*int)(nil)"
}
return fmt.Sprintf("%d", *i)
}
// IntPresent returns a boolean indiciating if the pointer is nil, or if the
// pointer is pointing to the zero value.
func IntPresent(i *int) bool {
if i == nil {
return false
}
return *i != 0
}
// Signal returns a pointer to the given os.Signal.
func Signal(s os.Signal) *os.Signal {
return &s
}
// SignalVal returns the value of the os.Signal at the pointer, or 0 if the
// pointer is nil.
func SignalVal(s *os.Signal) os.Signal {
if s == nil {
return (os.Signal)(nil)
}
return *s
}
// SignalGoString returns the value of the os.Signal for printing in a string.
func SignalGoString(s *os.Signal) string {
if s == nil {
return "(*os.Signal)(nil)"
}
if *s == nil {
return "<nil>"
}
return fmt.Sprintf("%q", *s)
}
// SignalPresent returns a boolean indiciating if the pointer is nil, or if the pointer is pointing to the zero value..
func SignalPresent(s *os.Signal) bool {
if s == nil {
return false
}
return *s != signals.SIGNIL
}
// String returns a pointer to the given string.
func String(s string) *string {
return &s
}
// StringVal returns the value of the string at the pointer, or "" if the
// pointer is nil.
func StringVal(s *string) string {
if s == nil {
return ""
}
return *s
}
// StringGoString returns the value of the string for printing in a string.
func StringGoString(s *string) string {
if s == nil {
return "(*string)(nil)"
}
return fmt.Sprintf("%q", *s)
}
// StringPresent returns a boolean indiciating if the pointer is nil, or if the pointer is pointing to the zero value..
func StringPresent(s *string) bool {
if s == nil {
return false
}
return *s != ""
}
// TimeDuration returns a pointer to the given time.Duration.
func TimeDuration(t time.Duration) *time.Duration {
return &t
}
// TimeDurationVal returns the value of the string at the pointer, or 0 if the
// pointer is nil.
func TimeDurationVal(t *time.Duration) time.Duration {
if t == nil {
return time.Duration(0)
}
return *t
}
// TimeDurationGoString returns the value of the time.Duration for printing in a
// string.
func TimeDurationGoString(t *time.Duration) string {
if t == nil {
return "(*time.Duration)(nil)"
}
return fmt.Sprintf("%s", t)
}
// TimeDurationPresent returns a boolean indiciating if the pointer is nil, or if the pointer is pointing to the zero value..
func TimeDurationPresent(t *time.Duration) bool {
if t == nil {
return false
}
return *t != 0
}

View File

@@ -0,0 +1,132 @@
package config
import (
"fmt"
"time"
)
const (
// DefaultDedupPrefix is the default prefix used for deduplication mode.
DefaultDedupPrefix = "consul-template/dedup/"
// DefaultDedupTTL is the default TTL for deduplicate mode.
DefaultDedupTTL = 15 * time.Second
// DefaultDedupMaxStale is the default max staleness for the deduplication
// manager.
DefaultDedupMaxStale = DefaultMaxStale
)
// DedupConfig is used to enable the de-duplication mode, which depends
// on electing a leader per-template and watching of a key. This is used
// to reduce the cost of many instances of CT running the same template.
type DedupConfig struct {
// Controls if deduplication mode is enabled
Enabled *bool `mapstructure:"enabled"`
// MaxStale is the maximum amount of time to allow for stale queries.
MaxStale *time.Duration `mapstructure:"max_stale"`
// Controls the KV prefix used. Defaults to defaultDedupPrefix
Prefix *string `mapstructure:"prefix"`
// TTL is the Session TTL used for lock acquisition, defaults to 15 seconds.
TTL *time.Duration `mapstructure:"ttl"`
}
// DefaultDedupConfig returns a configuration that is populated with the
// default values.
func DefaultDedupConfig() *DedupConfig {
return &DedupConfig{}
}
// Copy returns a deep copy of this configuration.
func (c *DedupConfig) Copy() *DedupConfig {
if c == nil {
return nil
}
var o DedupConfig
o.Enabled = c.Enabled
o.MaxStale = c.MaxStale
o.Prefix = c.Prefix
o.TTL = c.TTL
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *DedupConfig) Merge(o *DedupConfig) *DedupConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Enabled != nil {
r.Enabled = o.Enabled
}
if o.MaxStale != nil {
r.MaxStale = o.MaxStale
}
if o.Prefix != nil {
r.Prefix = o.Prefix
}
if o.TTL != nil {
r.TTL = o.TTL
}
return r
}
// Finalize ensures there no nil pointers.
func (c *DedupConfig) Finalize() {
if c.Enabled == nil {
c.Enabled = Bool(false ||
TimeDurationPresent(c.MaxStale) ||
StringPresent(c.Prefix) ||
TimeDurationPresent(c.TTL))
}
if c.MaxStale == nil {
c.MaxStale = TimeDuration(DefaultDedupMaxStale)
}
if c.Prefix == nil {
c.Prefix = String(DefaultDedupPrefix)
}
if c.TTL == nil {
c.TTL = TimeDuration(DefaultDedupTTL)
}
}
// GoString defines the printable version of this struct.
func (c *DedupConfig) GoString() string {
if c == nil {
return "(*DedupConfig)(nil)"
}
return fmt.Sprintf("&DedupConfig{"+
"Enabled:%s, "+
"MaxStale:%s, "+
"Prefix:%s, "+
"TTL:%s"+
"}",
BoolGoString(c.Enabled),
TimeDurationGoString(c.MaxStale),
StringGoString(c.Prefix),
TimeDurationGoString(c.TTL),
)
}

View File

@@ -0,0 +1,209 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// EnvConfig is an embeddable struct for things that accept environment
// variable filtering. You should not use this directly and it is only public
// for mapstructure's decoding.
type EnvConfig struct {
// BlacklistEnv specifies a list of environment variables to explicitly
// disclude from the list of environment variables populated to the child.
// If both WhitelistEnv and BlacklistEnv are provided, BlacklistEnv takes
// precedence over the values in WhitelistEnv.
Blacklist []string `mapstructure:"blacklist"`
// CustomEnv specifies custom environment variables to pass to the child
// process. These are provided programatically, override any environment
// variables of the same name, are ignored from whitelist/blacklist, and
// are still included even if PristineEnv is set to true.
Custom []string `mapstructure:"custom"`
// PristineEnv specifies if the child process should inherit the parent's
// environment.
Pristine *bool `mapstructure:"pristine"`
// WhitelistEnv specifies a list of environment variables to exclusively
// include in the list of environment variables populated to the child.
Whitelist []string `mapstructure:"whitelist"`
}
// DefaultEnvConfig returns a configuration that is populated with the
// default values.
func DefaultEnvConfig() *EnvConfig {
return &EnvConfig{}
}
// Copy returns a deep copy of this configuration.
func (c *EnvConfig) Copy() *EnvConfig {
if c == nil {
return nil
}
var o EnvConfig
if c.Blacklist != nil {
o.Blacklist = append([]string{}, c.Blacklist...)
}
if c.Custom != nil {
o.Custom = append([]string{}, c.Custom...)
}
o.Pristine = c.Pristine
if c.Whitelist != nil {
o.Whitelist = append([]string{}, c.Whitelist...)
}
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *EnvConfig) Merge(o *EnvConfig) *EnvConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Blacklist != nil {
r.Blacklist = append(r.Blacklist, o.Blacklist...)
}
if o.Custom != nil {
r.Custom = append(r.Custom, o.Custom...)
}
if o.Pristine != nil {
r.Pristine = o.Pristine
}
if o.Whitelist != nil {
r.Whitelist = append(r.Whitelist, o.Whitelist...)
}
return r
}
// Env calculates and returns the finalized environment for this exec
// configuration. It takes into account pristine, custom environment, whitelist,
// and blacklist values.
func (c *EnvConfig) Env() []string {
// In pristine mode, just return the custom environment. If the user did not
// specify a custom environment, just return the empty slice to force an
// empty environment. We cannot return nil here because the later call to
// os/exec will think we want to inherit the parent.
if BoolVal(c.Pristine) {
if len(c.Custom) > 0 {
return c.Custom
}
return []string{}
}
// Pull all the key-value pairs out of the environment
environ := os.Environ()
keys := make([]string, len(environ))
env := make(map[string]string, len(environ))
for i, v := range environ {
list := strings.SplitN(v, "=", 2)
keys[i] = list[0]
env[list[0]] = list[1]
}
// anyGlobMatch is a helper function which checks if any of the given globs
// match the string.
anyGlobMatch := func(s string, patterns []string) bool {
for _, pattern := range patterns {
if matched, _ := filepath.Match(pattern, s); matched {
return true
}
}
return false
}
// Pull out any envvars that match the whitelist.
if len(c.Whitelist) > 0 {
newKeys := make([]string, 0, len(keys))
for _, k := range keys {
if anyGlobMatch(k, c.Whitelist) {
newKeys = append(newKeys, k)
}
}
keys = newKeys
}
// Remove any envvars that match the blacklist.
if len(c.Blacklist) > 0 {
newKeys := make([]string, 0, len(keys))
for _, k := range keys {
if !anyGlobMatch(k, c.Blacklist) {
newKeys = append(newKeys, k)
}
}
keys = newKeys
}
// Build the final list using only the filtered keys.
finalEnv := make([]string, 0, len(keys)+len(c.Custom))
for _, k := range keys {
finalEnv = append(finalEnv, k+"="+env[k])
}
// Append remaining custom environment.
finalEnv = append(finalEnv, c.Custom...)
return finalEnv
}
// Finalize ensures there no nil pointers.
func (c *EnvConfig) Finalize() {
if c.Blacklist == nil {
c.Blacklist = []string{}
}
if c.Custom == nil {
c.Custom = []string{}
}
if c.Pristine == nil {
c.Pristine = Bool(false)
}
if c.Whitelist == nil {
c.Whitelist = []string{}
}
}
// GoString defines the printable version of this struct.
func (c *EnvConfig) GoString() string {
if c == nil {
return "(*EnvConfig)(nil)"
}
return fmt.Sprintf("&EnvConfig{"+
"Blacklist:%v, "+
"Custom:%v, "+
"Pristine:%s, "+
"Whitelist:%v"+
"}",
c.Blacklist,
c.Custom,
BoolGoString(c.Pristine),
c.Whitelist,
)
}

View File

@@ -0,0 +1,216 @@
package config
import (
"fmt"
"os"
"syscall"
"time"
)
const (
// DefaultExecKillSignal is the default signal to send to the process to
// tell it to gracefully terminate.
DefaultExecKillSignal = syscall.SIGINT
// DefaultExecKillTimeout is the maximum amount of time to wait for the
// process to gracefully terminate before force-killing it.
DefaultExecKillTimeout = 30 * time.Second
// DefaultExecTimeout is the default amount of time to wait for a
// command to exit. By default, this is disabled, which means the command
// is allowed to run for an infinite amount of time.
DefaultExecTimeout = 0 * time.Second
)
var (
// DefaultExecReloadSignal is the default signal to send to the process to
// tell it to reload its configuration.
DefaultExecReloadSignal = (os.Signal)(nil)
)
// ExecConfig is used to configure the application when it runs in
// exec/supervise mode.
type ExecConfig struct {
// Command is the command to execute and watch as a child process.
Command *string `mapstructure:"command"`
// Enabled controls if this exec is enabled.
Enabled *bool `mapstructure:"enabled"`
// EnvConfig is the environmental customizations.
Env *EnvConfig `mapstructure:"env"`
// KillSignal is the signal to send to the command to kill it gracefully. The
// default value is "SIGTERM".
KillSignal *os.Signal `mapstructure:"kill_signal"`
// KillTimeout is the amount of time to give the process to cleanup before
// hard-killing it.
KillTimeout *time.Duration `mapstructure:"kill_timeout"`
// ReloadSignal is the signal to send to the child process when a template
// changes. This tells the child process that templates have
ReloadSignal *os.Signal `mapstructure:"reload_signal"`
// Splay is the maximum amount of random time to wait to signal or kill the
// process. By default this is disabled, but it can be set to low values to
// reduce the "thundering herd" problem where all tasks are restarted at once.
Splay *time.Duration `mapstructure:"splay"`
// Timeout is the maximum amount of time to wait for a command to complete.
// By default, this is 0, which means "wait forever".
Timeout *time.Duration `mapstructure:"timeout"`
}
// DefaultExecConfig returns a configuration that is populated with the
// default values.
func DefaultExecConfig() *ExecConfig {
return &ExecConfig{
Env: DefaultEnvConfig(),
}
}
// Copy returns a deep copy of this configuration.
func (c *ExecConfig) Copy() *ExecConfig {
if c == nil {
return nil
}
var o ExecConfig
o.Command = c.Command
o.Enabled = c.Enabled
if c.Env != nil {
o.Env = c.Env.Copy()
}
o.KillSignal = c.KillSignal
o.KillTimeout = c.KillTimeout
o.ReloadSignal = c.ReloadSignal
o.Splay = c.Splay
o.Timeout = c.Timeout
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *ExecConfig) Merge(o *ExecConfig) *ExecConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Command != nil {
r.Command = o.Command
}
if o.Enabled != nil {
r.Enabled = o.Enabled
}
if o.Env != nil {
r.Env = r.Env.Merge(o.Env)
}
if o.KillSignal != nil {
r.KillSignal = o.KillSignal
}
if o.KillTimeout != nil {
r.KillTimeout = o.KillTimeout
}
if o.ReloadSignal != nil {
r.ReloadSignal = o.ReloadSignal
}
if o.Splay != nil {
r.Splay = o.Splay
}
if o.Timeout != nil {
r.Timeout = o.Timeout
}
return r
}
// Finalize ensures there no nil pointers.
func (c *ExecConfig) Finalize() {
if c.Enabled == nil {
c.Enabled = Bool(StringPresent(c.Command))
}
if c.Command == nil {
c.Command = String("")
}
if c.Env == nil {
c.Env = DefaultEnvConfig()
}
c.Env.Finalize()
if c.KillSignal == nil {
c.KillSignal = Signal(DefaultExecKillSignal)
}
if c.KillTimeout == nil {
c.KillTimeout = TimeDuration(DefaultExecKillTimeout)
}
if c.ReloadSignal == nil {
c.ReloadSignal = Signal(DefaultExecReloadSignal)
}
if c.Splay == nil {
c.Splay = TimeDuration(0 * time.Second)
}
if c.Timeout == nil {
c.Timeout = TimeDuration(DefaultExecTimeout)
}
}
// GoString defines the printable version of this struct.
func (c *ExecConfig) GoString() string {
if c == nil {
return "(*ExecConfig)(nil)"
}
return fmt.Sprintf("&ExecConfig{"+
"Command:%s, "+
"Enabled:%s, "+
"Env:%#v, "+
"KillSignal:%s, "+
"KillTimeout:%s, "+
"ReloadSignal:%s, "+
"Splay:%s, "+
"Timeout:%s"+
"}",
StringGoString(c.Command),
BoolGoString(c.Enabled),
c.Env,
SignalGoString(c.KillSignal),
TimeDurationGoString(c.KillTimeout),
SignalGoString(c.ReloadSignal),
TimeDurationGoString(c.Splay),
TimeDurationGoString(c.Timeout),
)
}

View File

@@ -1,6 +1,7 @@
package config
import (
"log"
"os"
"reflect"
"strconv"
@@ -31,3 +32,44 @@ func StringToFileModeFunc() mapstructure.DecodeHookFunc {
return os.FileMode(v), nil
}
}
// StringToWaitDurationHookFunc returns a function that converts strings to wait
// value. This is designed to be used with mapstructure for parsing out a wait
// value.
func StringToWaitDurationHookFunc() mapstructure.DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(WaitConfig{}) {
return data, nil
}
// Convert it by parsing
return ParseWaitConfig(data.(string))
}
}
// ConsulStringToStructFunc checks if the value set for the key should actually
// be a struct and sets the appropriate value in the struct. This is for
// backwards-compatability with older versions of Consul Template.
func ConsulStringToStructFunc() mapstructure.DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if t == reflect.TypeOf(ConsulConfig{}) && f.Kind() == reflect.String {
log.Println("[WARN] consul now accepts a stanza instead of a string. " +
"Update your configuration files and change consul = \"\" to " +
"consul { } instead.")
return &ConsulConfig{
Address: String(data.(string)),
}, nil
}
return data, nil
}
}

View File

@@ -0,0 +1,140 @@
package config
import (
"fmt"
"math"
"time"
)
const (
// DefaultRetryAttempts is the default number of maximum retry attempts.
DefaultRetryAttempts = 5
// DefaultRetryBackoff is the default base for the exponential backoff
// algorithm.
DefaultRetryBackoff = 250 * time.Millisecond
)
// RetryFunc is the signature of a function that supports retries.
type RetryFunc func(int) (bool, time.Duration)
// RetryConfig is a shared configuration for upstreams that support retires on
// failure.
type RetryConfig struct {
// Attempts is the total number of maximum attempts to retry before letting
// the error fall through.
Attempts *int
// Backoff is the base of the exponentialbackoff. This number will be
// multipled by the next power of 2 on each iteration.
Backoff *time.Duration
// Enabled signals if this retry is enabled.
Enabled *bool
}
// DefaultRetryConfig returns a configuration that is populated with the
// default values.
func DefaultRetryConfig() *RetryConfig {
return &RetryConfig{}
}
// Copy returns a deep copy of this configuration.
func (c *RetryConfig) Copy() *RetryConfig {
if c == nil {
return nil
}
var o RetryConfig
o.Attempts = c.Attempts
o.Backoff = c.Backoff
o.Enabled = c.Enabled
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *RetryConfig) Merge(o *RetryConfig) *RetryConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Attempts != nil {
r.Attempts = o.Attempts
}
if o.Backoff != nil {
r.Backoff = o.Backoff
}
if o.Enabled != nil {
r.Enabled = o.Enabled
}
return r
}
// RetryFunc returns the retry function associated with this configuration.
func (c *RetryConfig) RetryFunc() RetryFunc {
return func(retry int) (bool, time.Duration) {
if !BoolVal(c.Enabled) {
return false, 0
}
if IntVal(c.Attempts) > 0 && retry > IntVal(c.Attempts)-1 {
return false, 0
}
base := math.Pow(2, float64(retry))
sleep := time.Duration(base) * TimeDurationVal(c.Backoff)
return true, sleep
}
}
// Finalize ensures there no nil pointers.
func (c *RetryConfig) Finalize() {
if c.Attempts == nil {
c.Attempts = Int(DefaultRetryAttempts)
}
if c.Backoff == nil {
c.Backoff = TimeDuration(DefaultRetryBackoff)
}
if c.Enabled == nil {
c.Enabled = Bool(true)
}
}
// GoString defines the printable version of this struct.
func (c *RetryConfig) GoString() string {
if c == nil {
return "(*RetryConfig)(nil)"
}
return fmt.Sprintf("&RetryConfig{"+
"Attempts:%s, "+
"Backoff:%s, "+
"Enabled:%s"+
"}",
IntGoString(c.Attempts),
TimeDurationGoString(c.Backoff),
BoolGoString(c.Enabled),
)
}

View File

@@ -0,0 +1,153 @@
package config
import "fmt"
const (
// DefaultSSLVerify is the default value for SSL verification.
DefaultSSLVerify = true
)
// SSLConfig is the configuration for SSL.
type SSLConfig struct {
CaCert *string `mapstructure:"ca_cert"`
CaPath *string `mapstructure:"ca_path"`
Cert *string `mapstructure:"cert"`
Enabled *bool `mapstructure:"enabled"`
Key *string `mapstructure:"key"`
ServerName *string `mapstructure:"server_name"`
Verify *bool `mapstructure:"verify"`
}
// DefaultSSLConfig returns a configuration that is populated with the
// default values.
func DefaultSSLConfig() *SSLConfig {
return &SSLConfig{}
}
// Copy returns a deep copy of this configuration.
func (c *SSLConfig) Copy() *SSLConfig {
if c == nil {
return nil
}
var o SSLConfig
o.CaCert = c.CaCert
o.CaPath = c.CaPath
o.Cert = c.Cert
o.Enabled = c.Enabled
o.Key = c.Key
o.ServerName = c.ServerName
o.Verify = c.Verify
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *SSLConfig) Merge(o *SSLConfig) *SSLConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Cert != nil {
r.Cert = o.Cert
}
if o.CaCert != nil {
r.CaCert = o.CaCert
}
if o.CaPath != nil {
r.CaPath = o.CaPath
}
if o.Enabled != nil {
r.Enabled = o.Enabled
}
if o.Key != nil {
r.Key = o.Key
}
if o.ServerName != nil {
r.ServerName = o.ServerName
}
if o.Verify != nil {
r.Verify = o.Verify
}
return r
}
// Finalize ensures there no nil pointers.
func (c *SSLConfig) Finalize() {
if c.Enabled == nil {
c.Enabled = Bool(false ||
StringPresent(c.Cert) ||
StringPresent(c.CaCert) ||
StringPresent(c.CaPath) ||
StringPresent(c.Key) ||
StringPresent(c.ServerName) ||
BoolPresent(c.Verify))
}
if c.Cert == nil {
c.Cert = String("")
}
if c.CaCert == nil {
c.CaCert = String("")
}
if c.CaPath == nil {
c.CaPath = String("")
}
if c.Key == nil {
c.Key = String("")
}
if c.ServerName == nil {
c.ServerName = String("")
}
if c.Verify == nil {
c.Verify = Bool(DefaultSSLVerify)
}
}
// GoString defines the printable version of this struct.
func (c *SSLConfig) GoString() string {
if c == nil {
return "(*SSLConfig)(nil)"
}
return fmt.Sprintf("&SSLConfig{"+
"CaCert:%s, "+
"CaPath:%s, "+
"Cert:%s, "+
"Enabled:%s, "+
"Key:%s, "+
"ServerName:%s, "+
"Verify:%s"+
"}",
StringGoString(c.CaCert),
StringGoString(c.CaPath),
StringGoString(c.Cert),
BoolGoString(c.Enabled),
StringGoString(c.Key),
StringGoString(c.ServerName),
BoolGoString(c.Verify),
)
}

View File

@@ -0,0 +1,87 @@
package config
import "fmt"
const (
// DefaultSyslogFacility is the default facility to log to.
DefaultSyslogFacility = "LOCAL0"
)
// SyslogConfig is the configuration for syslog.
type SyslogConfig struct {
Enabled *bool `mapstructure:"enabled"`
Facility *string `mapstructure:"facility"`
}
// DefaultSyslogConfig returns a configuration that is populated with the
// default values.
func DefaultSyslogConfig() *SyslogConfig {
return &SyslogConfig{}
}
// Copy returns a deep copy of this configuration.
func (c *SyslogConfig) Copy() *SyslogConfig {
if c == nil {
return nil
}
var o SyslogConfig
o.Enabled = c.Enabled
o.Facility = c.Facility
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *SyslogConfig) Merge(o *SyslogConfig) *SyslogConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Enabled != nil {
r.Enabled = o.Enabled
}
if o.Facility != nil {
r.Facility = o.Facility
}
return r
}
// Finalize ensures there no nil pointers.
func (c *SyslogConfig) Finalize() {
if c.Enabled == nil {
c.Enabled = Bool(StringPresent(c.Facility))
}
if c.Facility == nil {
c.Facility = String(DefaultSyslogFacility)
}
}
// GoString defines the printable version of this struct.
func (c *SyslogConfig) GoString() string {
if c == nil {
return "(*SyslogConfig)(nil)"
}
return fmt.Sprintf("&SyslogConfig{"+
"Enabled:%s, "+
"Facility:%s"+
"}",
BoolGoString(c.Enabled),
StringGoString(c.Facility),
)
}

View File

@@ -0,0 +1,405 @@
package config
import (
"errors"
"fmt"
"os"
"regexp"
"strings"
"time"
)
const (
// DefaultTemplateFilePerms are the default file permissions for templates
// rendered onto disk when a specific file permission has not already been
// specified.
DefaultTemplateFilePerms = 0644
// DefaultTemplateCommandTimeout is the amount of time to wait for a command
// to return.
DefaultTemplateCommandTimeout = 30 * time.Second
)
var (
// ErrTemplateStringEmpty is the error returned with the template contents
// are empty.
ErrTemplateStringEmpty = errors.New("template: cannot be empty")
// ErrTemplateInvalidFormat is the error returned with the template is not
// a valid format.
ErrTemplateInvalidFormat = errors.New("template: invalid format")
// configTemplateRe is the pattern to split the config template syntax.
configTemplateRe = regexp.MustCompile("([a-zA-Z]:)?([^:]+)")
)
// TemplateConfig is a representation of a template on disk, as well as the
// associated commands and reload instructions.
type TemplateConfig struct {
// Backup determines if this template should retain a backup. The default
// value is false.
Backup *bool `mapstructure:"backup"`
// Command is the arbitrary command to execute after a template has
// successfully rendered. This is DEPRECATED. Use Exec instead.
Command *string `mapstructure:"command"`
// CommandTimeout is the amount of time to wait for the command to finish
// before force-killing it. This is DEPRECATED. Use Exec instead.
CommandTimeout *time.Duration `mapstructure:"command_timeout"`
// Contents are the raw template contents to evaluate. Either this or Source
// must be specified, but not both.
Contents *string `mapstructure:"contents"`
// Destination is the location on disk where the template should be rendered.
// This is required unless running in debug/dry mode.
Destination *string `mapstructure:"destination"`
// Exec is the configuration for the command to run when the template renders
// successfully.
Exec *ExecConfig `mapstructure:"exec"`
// Perms are the file system permissions to use when creating the file on
// disk. This is useful for when files contain sensitive information, such as
// secrets from Vault.
Perms *os.FileMode `mapstructure:"perms"`
// Source is the path on disk to the template contents to evaluate. Either
// this or Contents should be specified, but not both.
Source *string `mapstructure:"source"`
// Wait configures per-template quiescence timers.
Wait *WaitConfig `mapstructure:"wait"`
// LeftDelim and RightDelim are optional configurations to control what
// delimiter is utilized when parsing the template.
LeftDelim *string `mapstructure:"left_delimiter"`
RightDelim *string `mapstructure:"right_delimiter"`
}
// DefaultTemplateConfig returns a configuration that is populated with the
// default values.
func DefaultTemplateConfig() *TemplateConfig {
return &TemplateConfig{
Exec: DefaultExecConfig(),
Wait: DefaultWaitConfig(),
}
}
// Copy returns a deep copy of this configuration.
func (c *TemplateConfig) Copy() *TemplateConfig {
if c == nil {
return nil
}
var o TemplateConfig
o.Backup = c.Backup
o.Command = c.Command
o.CommandTimeout = c.CommandTimeout
o.Contents = c.Contents
o.Destination = c.Destination
if c.Exec != nil {
o.Exec = c.Exec.Copy()
}
o.Perms = c.Perms
o.Source = c.Source
if c.Wait != nil {
o.Wait = c.Wait.Copy()
}
o.LeftDelim = c.LeftDelim
o.RightDelim = c.RightDelim
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *TemplateConfig) Merge(o *TemplateConfig) *TemplateConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Backup != nil {
r.Backup = o.Backup
}
if o.Command != nil {
r.Command = o.Command
}
if o.CommandTimeout != nil {
r.CommandTimeout = o.CommandTimeout
}
if o.Contents != nil {
r.Contents = o.Contents
}
if o.Destination != nil {
r.Destination = o.Destination
}
if o.Exec != nil {
r.Exec = r.Exec.Merge(o.Exec)
}
if o.Perms != nil {
r.Perms = o.Perms
}
if o.Source != nil {
r.Source = o.Source
}
if o.Wait != nil {
r.Wait = r.Wait.Merge(o.Wait)
}
if o.LeftDelim != nil {
r.LeftDelim = o.LeftDelim
}
if o.RightDelim != nil {
r.RightDelim = o.RightDelim
}
return r
}
// Finalize ensures the configuration has no nil pointers and sets default
// values.
func (c *TemplateConfig) Finalize() {
if c.Backup == nil {
c.Backup = Bool(false)
}
if c.Command == nil {
c.Command = String("")
}
if c.CommandTimeout == nil {
c.CommandTimeout = TimeDuration(DefaultTemplateCommandTimeout)
}
if c.Contents == nil {
c.Contents = String("")
}
if c.Destination == nil {
c.Destination = String("")
}
if c.Exec == nil {
c.Exec = DefaultExecConfig()
}
// Backwards compat for specifying command directly
if c.Exec.Command == nil && c.Command != nil {
c.Exec.Command = c.Command
}
if c.Exec.Timeout == nil && c.CommandTimeout != nil {
c.Exec.Timeout = c.CommandTimeout
}
c.Exec.Finalize()
if c.Perms == nil {
c.Perms = FileMode(DefaultTemplateFilePerms)
}
if c.Source == nil {
c.Source = String("")
}
if c.Wait == nil {
c.Wait = DefaultWaitConfig()
}
c.Wait.Finalize()
if c.LeftDelim == nil {
c.LeftDelim = String("")
}
if c.RightDelim == nil {
c.RightDelim = String("")
}
}
// GoString defines the printable version of this struct.
func (c *TemplateConfig) GoString() string {
if c == nil {
return "(*TemplateConfig)(nil)"
}
return fmt.Sprintf("&TemplateConfig{"+
"Backup:%s, "+
"Command:%s, "+
"CommandTimeout:%s, "+
"Contents:%s, "+
"Destination:%s, "+
"Exec:%#v, "+
"Perms:%s, "+
"Source:%s, "+
"Wait:%#v, "+
"LeftDelim:%s, "+
"RightDelim:%s"+
"}",
BoolGoString(c.Backup),
StringGoString(c.Command),
TimeDurationGoString(c.CommandTimeout),
StringGoString(c.Contents),
StringGoString(c.Destination),
c.Exec,
FileModeGoString(c.Perms),
StringGoString(c.Source),
c.Wait,
StringGoString(c.LeftDelim),
StringGoString(c.RightDelim),
)
}
// Display is the human-friendly form of this configuration. It tries to
// describe this template in as much detail as possible in a single line, so
// log consumers can uniquely identify it.
func (c *TemplateConfig) Display() string {
if c == nil {
return ""
}
source := c.Source
if StringPresent(c.Contents) {
source = String("(dynamic)")
}
return fmt.Sprintf("%q => %q",
StringVal(source),
StringVal(c.Destination),
)
}
// TemplateConfigs is a collection of TemplateConfigs
type TemplateConfigs []*TemplateConfig
// DefaultTemplateConfigs returns a configuration that is populated with the
// default values.
func DefaultTemplateConfigs() *TemplateConfigs {
return &TemplateConfigs{}
}
// Copy returns a deep copy of this configuration.
func (c *TemplateConfigs) Copy() *TemplateConfigs {
o := make(TemplateConfigs, len(*c))
for i, t := range *c {
o[i] = t.Copy()
}
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *TemplateConfigs) Merge(o *TemplateConfigs) *TemplateConfigs {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
*r = append(*r, *o...)
return r
}
// Finalize ensures the configuration has no nil pointers and sets default
// values.
func (c *TemplateConfigs) Finalize() {
if c == nil {
*c = *DefaultTemplateConfigs()
}
for _, t := range *c {
t.Finalize()
}
}
// GoString defines the printable version of this struct.
func (c *TemplateConfigs) GoString() string {
if c == nil {
return "(*TemplateConfigs)(nil)"
}
s := make([]string, len(*c))
for i, t := range *c {
s[i] = t.GoString()
}
return "{" + strings.Join(s, ", ") + "}"
}
// ParseTemplateConfig parses a string in the form source:destination:command
// into a TemplateConfig.
func ParseTemplateConfig(s string) (*TemplateConfig, error) {
if len(strings.TrimSpace(s)) < 1 {
return nil, ErrTemplateStringEmpty
}
var source, destination, command string
parts := configTemplateRe.FindAllString(s, -1)
switch len(parts) {
case 1:
source = parts[0]
case 2:
source, destination = parts[0], parts[1]
case 3:
source, destination, command = parts[0], parts[1], parts[2]
default:
return nil, ErrTemplateInvalidFormat
}
var sourcePtr, destinationPtr, commandPtr *string
if source != "" {
sourcePtr = String(source)
}
if destination != "" {
destinationPtr = String(destination)
}
if command != "" {
commandPtr = String(command)
}
return &TemplateConfig{
Source: sourcePtr,
Destination: destinationPtr,
Command: commandPtr,
}, nil
}

View File

@@ -0,0 +1,212 @@
package config
import (
"fmt"
"time"
"github.com/hashicorp/vault/api"
)
const (
// DefaultVaultRenewToken is the default value for if the Vault token should
// be renewed.
DefaultVaultRenewToken = true
// DefaultVaultUnwrapToken is the default value for if the Vault token should
// be unwrapped.
DefaultVaultUnwrapToken = false
// DefaultVaultRetryBase is the default value for the base time to use for
// exponential backoff.
DefaultVaultRetryBase = 250 * time.Millisecond
// DefaultVaultRetryMaxAttempts is the default maximum number of attempts to
// retry before quitting.
DefaultVaultRetryMaxAttempts = 5
)
// VaultConfig is the configuration for connecting to a vault server.
type VaultConfig struct {
// Address is the URI to the Vault server.
Address *string `mapstructure:"address"`
// Enabled controls whether the Vault integration is active.
Enabled *bool `mapstructure:"enabled"`
// RenewToken renews the Vault token.
RenewToken *bool `mapstructure:"renew_token"`
// Retry is the configuration for specifying how to behave on failure.
Retry *RetryConfig `mapstructure:"retry"`
// SSL indicates we should use a secure connection while talking to Vault.
SSL *SSLConfig `mapstructure:"ssl"`
// Token is the Vault token to communicate with for requests. It may be
// a wrapped token or a real token. This can also be set via the VAULT_TOKEN
// environment variable.
Token *string `mapstructure:"token" json:"-"`
// UnwrapToken unwraps the provided Vault token as a wrapped token.
UnwrapToken *bool `mapstructure:"unwrap_token"`
}
// DefaultVaultConfig returns a configuration that is populated with the
// default values.
func DefaultVaultConfig() *VaultConfig {
v := &VaultConfig{
Address: stringFromEnv(api.EnvVaultAddress),
RenewToken: boolFromEnv("VAULT_RENEW_TOKEN"),
UnwrapToken: boolFromEnv("VAULT_UNWRAP_TOKEN"),
Retry: DefaultRetryConfig(),
SSL: &SSLConfig{
CaCert: stringFromEnv(api.EnvVaultCACert),
CaPath: stringFromEnv(api.EnvVaultCAPath),
Cert: stringFromEnv(api.EnvVaultClientCert),
Key: stringFromEnv(api.EnvVaultClientKey),
ServerName: stringFromEnv(api.EnvVaultTLSServerName),
Verify: antiboolFromEnv(api.EnvVaultInsecure),
},
Token: stringFromEnv("VAULT_TOKEN"),
}
// Force SSL when communicating with Vault.
v.SSL.Enabled = Bool(true)
return v
}
// Copy returns a deep copy of this configuration.
func (c *VaultConfig) Copy() *VaultConfig {
if c == nil {
return nil
}
var o VaultConfig
o.Address = c.Address
o.Enabled = c.Enabled
o.RenewToken = c.RenewToken
if c.Retry != nil {
o.Retry = c.Retry.Copy()
}
if c.SSL != nil {
o.SSL = c.SSL.Copy()
}
o.Token = c.Token
o.UnwrapToken = c.UnwrapToken
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *VaultConfig) Merge(o *VaultConfig) *VaultConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Address != nil {
r.Address = o.Address
}
if o.Enabled != nil {
r.Enabled = o.Enabled
}
if o.RenewToken != nil {
r.RenewToken = o.RenewToken
}
if o.Retry != nil {
r.Retry = r.Retry.Merge(o.Retry)
}
if o.SSL != nil {
r.SSL = r.SSL.Merge(o.SSL)
}
if o.Token != nil {
r.Token = o.Token
}
if o.UnwrapToken != nil {
r.UnwrapToken = o.UnwrapToken
}
return r
}
// Finalize ensures there no nil pointers.
func (c *VaultConfig) Finalize() {
if c.Enabled == nil {
c.Enabled = Bool(StringPresent(c.Address))
}
if c.Address == nil {
c.Address = String("")
}
if c.RenewToken == nil {
c.RenewToken = Bool(DefaultVaultRenewToken)
}
if c.Retry == nil {
c.Retry = DefaultRetryConfig()
}
c.Retry.Finalize()
if c.SSL == nil {
c.SSL = DefaultSSLConfig()
}
c.SSL.Finalize()
if c.Token == nil {
c.Token = String("")
}
if c.UnwrapToken == nil {
c.UnwrapToken = Bool(DefaultVaultUnwrapToken)
}
}
// GoString defines the printable version of this struct.
func (c *VaultConfig) GoString() string {
if c == nil {
return "(*VaultConfig)(nil)"
}
return fmt.Sprintf("&VaultConfig{"+
"Enabled:%s, "+
"Address:%s, "+
"Token:%t, "+
"UnwrapToken:%s, "+
"RenewToken:%s, "+
"Retry:%#v, "+
"SSL:%#v"+
"}",
BoolGoString(c.Enabled),
StringGoString(c.Address),
StringPresent(c.Token),
BoolGoString(c.UnwrapToken),
BoolGoString(c.RenewToken),
c.Retry,
c.SSL,
)
}

View File

@@ -0,0 +1,191 @@
package config
import (
"errors"
"fmt"
"strings"
"time"
)
var (
// ErrWaitStringEmpty is the error returned when wait is specified as an empty
// string.
ErrWaitStringEmpty = errors.New("wait: cannot be empty")
// ErrWaitInvalidFormat is the error returned when the wait is specified
// incorrectly.
ErrWaitInvalidFormat = errors.New("wait: invalid format")
// ErrWaitNegative is the error returned with the wait is negative.
ErrWaitNegative = errors.New("wait: cannot be negative")
// ErrWaitMinLTMax is the error returned with the minimum wait time is not
// less than the maximum wait time.
ErrWaitMinLTMax = errors.New("wait: min must be less than max")
)
// WaitConfig is the Min/Max duration used by the Watcher
type WaitConfig struct {
// Enabled determines if this wait is enabled.
Enabled *bool `mapstructure:"bool"`
// Min and Max are the minimum and maximum time, respectively, to wait for
// data changes before rendering a new template to disk.
Min *time.Duration `mapstructure:"min"`
Max *time.Duration `mapstructure:"max"`
}
// DefaultWaitConfig is the default configuration.
func DefaultWaitConfig() *WaitConfig {
return &WaitConfig{}
}
// Copy returns a deep copy of this configuration.
func (c *WaitConfig) Copy() *WaitConfig {
if c == nil {
return nil
}
var o WaitConfig
o.Enabled = c.Enabled
o.Min = c.Min
o.Max = c.Max
return &o
}
// Merge combines all values in this configuration with the values in the other
// configuration, with values in the other configuration taking precedence.
// Maps and slices are merged, most other values are overwritten. Complex
// structs define their own merge functionality.
func (c *WaitConfig) Merge(o *WaitConfig) *WaitConfig {
if c == nil {
if o == nil {
return nil
}
return o.Copy()
}
if o == nil {
return c.Copy()
}
r := c.Copy()
if o.Enabled != nil {
r.Enabled = o.Enabled
}
if o.Min != nil {
r.Min = o.Min
}
if o.Max != nil {
r.Max = o.Max
}
return r
}
// Finalize ensures there no nil pointers.
func (c *WaitConfig) Finalize() {
if c.Enabled == nil {
c.Enabled = Bool(TimeDurationPresent(c.Min))
}
if c.Min == nil {
c.Min = TimeDuration(0 * time.Second)
}
if c.Max == nil {
c.Max = TimeDuration(4 * *c.Min)
}
}
// GoString defines the printable version of this struct.
func (c *WaitConfig) GoString() string {
if c == nil {
return "(*WaitConfig)(nil)"
}
return fmt.Sprintf("&WaitConfig{"+
"Enabled:%s, "+
"Min:%s, "+
"Max:%s"+
"}",
BoolGoString(c.Enabled),
TimeDurationGoString(c.Min),
TimeDurationGoString(c.Max),
)
}
// ParseWaitConfig parses a string of the format `minimum(:maximum)` into a
// WaitConfig.
func ParseWaitConfig(s string) (*WaitConfig, error) {
s = strings.TrimSpace(s)
if len(s) < 1 {
return nil, ErrWaitStringEmpty
}
parts := strings.Split(s, ":")
var min, max time.Duration
var err error
switch len(parts) {
case 1:
min, err = time.ParseDuration(strings.TrimSpace(parts[0]))
if err != nil {
return nil, err
}
max = 4 * min
case 2:
min, err = time.ParseDuration(strings.TrimSpace(parts[0]))
if err != nil {
return nil, err
}
max, err = time.ParseDuration(strings.TrimSpace(parts[1]))
if err != nil {
return nil, err
}
default:
return nil, ErrWaitInvalidFormat
}
if min < 0 || max < 0 {
return nil, ErrWaitNegative
}
if max < min {
return nil, ErrWaitMinLTMax
}
var c WaitConfig
c.Min = TimeDuration(min)
c.Max = TimeDuration(max)
return &c, nil
}
// WaitVar implements the Flag.Value interface and allows the user to specify
// a watch interval using Go's flag parsing library.
type WaitVar WaitConfig
// Set sets the value in the format min[:max] for a wait timer.
func (w *WaitVar) Set(value string) error {
wait, err := ParseWaitConfig(value)
if err != nil {
return err
}
w.Min = wait.Min
w.Max = wait.Max
return nil
}
// String returns the string format for this wait variable
func (w *WaitVar) String() string {
return fmt.Sprintf("%s:%s", w.Min, w.Max)
}

View File

@@ -0,0 +1,92 @@
package dependency
import (
"log"
"net/url"
"sort"
"time"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*CatalogDatacentersQuery)(nil)
// CatalogDatacentersQuerySleepTime is the amount of time to sleep between
// queries, since the endpoint does not support blocking queries.
CatalogDatacentersQuerySleepTime = 15 * time.Second
)
// CatalogDatacentersQuery is the dependency to query all datacenters
type CatalogDatacentersQuery struct {
stopCh chan struct{}
}
// NewCatalogDatacentersQuery creates a new datacenter dependency.
func NewCatalogDatacentersQuery() (*CatalogDatacentersQuery, error) {
return &CatalogDatacentersQuery{
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Consul API defined by the given client and returns a slice
// of strings representing the datacenters
func (d *CatalogDatacentersQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
opts = opts.Merge(&QueryOptions{})
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/catalog/datacenters",
RawQuery: opts.String(),
})
// This is pretty ghetto, but the datacenters endpoint does not support
// blocking queries, so we are going to "fake it until we make it". When we
// first query, the LastIndex will be "0", meaning we should immediately
// return data, but future calls will include a LastIndex. If we have a
// LastIndex in the query metadata, sleep for 15 seconds before asking Consul
// again.
//
// This is probably okay given the frequency in which datacenters actually
// change, but is technically not edge-triggering.
if opts.WaitIndex != 0 {
log.Printf("[TRACE] %s: long polling for %s", d, CatalogDatacentersQuerySleepTime)
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-time.After(CatalogDatacentersQuerySleepTime):
}
}
result, err := clients.Consul().Catalog().Datacenters()
if err != nil {
return nil, nil, errors.Wrapf(err, d.String())
}
log.Printf("[TRACE] %s: returned %d results", d, len(result))
sort.Strings(result)
return respWithMetadata(result)
}
// CanShare returns if this dependency is shareable.
func (d *CatalogDatacentersQuery) CanShare() bool {
return true
}
// String returns the human-friendly version of this dependency.
func (d *CatalogDatacentersQuery) String() string {
return "catalog.datacenters"
}
// Stop terminates this dependency's fetch.
func (d *CatalogDatacentersQuery) Stop() {
close(d.stopCh)
}
// Type returns the type of this dependency.
func (d *CatalogDatacentersQuery) Type() Type {
return TypeConsul
}

View File

@@ -2,29 +2,44 @@ package dependency
import (
"encoding/gob"
"errors"
"fmt"
"log"
"net/url"
"regexp"
"sort"
"sync"
"github.com/hashicorp/consul/api"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*CatalogNodeQuery)(nil)
// CatalogNodeQueryRe is the regular expression to use.
CatalogNodeQueryRe = regexp.MustCompile(`\A` + nameRe + dcRe + `\z`)
)
func init() {
gob.Register([]*NodeDetail{})
gob.Register([]*NodeService{})
gob.Register([]*CatalogNode{})
gob.Register([]*CatalogNodeService{})
}
// NodeDetail is a wrapper around the node and its services.
type NodeDetail struct {
// CatalogNodeQuery represents a single node from the Consul catalog.
type CatalogNodeQuery struct {
stopCh chan struct{}
dc string
name string
}
// CatalogNode is a wrapper around the node and its services.
type CatalogNode struct {
Node *Node
Services NodeServiceList
Services []*CatalogNodeService
}
// NodeService is a service on a single node.
type NodeService struct {
// CatalogNodeService is a service on a single node.
type CatalogNodeService struct {
ID string
Service string
Tags ServiceTags
@@ -33,82 +48,68 @@ type NodeService struct {
EnableTagOverride bool
}
// CatalogNode represents a single node from the Consul catalog.
type CatalogNode struct {
sync.Mutex
// NewCatalogNodeQuery parses the given string into a dependency. If the name is
// empty then the name of the local agent is used.
func NewCatalogNodeQuery(s string) (*CatalogNodeQuery, error) {
if s != "" && !CatalogNodeQueryRe.MatchString(s) {
return nil, fmt.Errorf("catalog.node: invalid format: %q", s)
}
rawKey string
dataCenter string
stopped bool
stopCh chan struct{}
m := regexpMatch(CatalogNodeQueryRe, s)
return &CatalogNodeQuery{
dc: m["dc"],
name: m["name"],
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Consul API defined by the given client and returns a
// of NodeDetail object.
func (d *CatalogNode) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
consulOpts := opts.consulQueryOptions()
if d.dataCenter != "" {
consulOpts.Datacenter = d.dataCenter
}
consul, err := clients.Consul()
if err != nil {
return nil, nil, fmt.Errorf("catalog node: error getting client: %s", err)
}
nodeName := d.rawKey
if nodeName == "" {
log.Printf("[DEBUG] (%s) getting local agent name", d.Display())
nodeName, err = consul.Agent().NodeName()
if err != nil {
return nil, nil, fmt.Errorf("catalog node: error getting local agent: %s", err)
}
}
var n *api.CatalogNode
var qm *api.QueryMeta
dataCh := make(chan struct{})
go func() {
log.Printf("[DEBUG] (%s) querying consul with %+v", d.Display(), consulOpts)
n, qm, err = consul.Catalog().Node(nodeName, consulOpts)
close(dataCh)
}()
// of CatalogNode object.
func (d *CatalogNodeQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-dataCh:
default:
}
if err != nil {
return nil, nil, fmt.Errorf("catalog node: error fetching: %s", err)
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
})
if d.name == "" {
log.Printf("[TRACE] %s: getting local agent name", d)
name, err := clients.Consul().Agent().NodeName()
if err != nil {
return nil, nil, errors.Wrapf(err, d.String())
}
d.name = name
}
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/catalog/node/" + d.name,
RawQuery: opts.String(),
})
node, qm, err := clients.Consul().Catalog().Node(d.name, opts.ToConsulOpts())
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
log.Printf("[TRACE] %s: returned response", d)
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
LastContact: qm.LastContact,
}
if n == nil {
log.Printf("[WARN] (%s) could not find node by that name", d.Display())
var node *NodeDetail
return node, rm, nil
if node == nil {
log.Printf("[WARN] %s: no node exists with the name %q", d, d.name)
var node CatalogNode
return &node, rm, nil
}
services := make(NodeServiceList, 0, len(n.Services))
for _, v := range n.Services {
services = append(services, &NodeService{
services := make([]*CatalogNodeService, 0, len(node.Services))
for _, v := range node.Services {
services = append(services, &CatalogNodeService{
ID: v.ID,
Service: v.Service,
Tags: ServiceTags(deepCopyAndSortTags(v.Tags)),
@@ -117,107 +118,54 @@ func (d *CatalogNode) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}
EnableTagOverride: v.EnableTagOverride,
})
}
sort.Stable(services)
sort.Stable(ByService(services))
node := &NodeDetail{
detail := &CatalogNode{
Node: &Node{
Node: n.Node.Node,
Address: n.Node.Address,
TaggedAddresses: n.Node.TaggedAddresses,
Node: node.Node.Node,
Address: node.Node.Address,
TaggedAddresses: node.Node.TaggedAddresses,
},
Services: services,
}
return node, rm, nil
return detail, rm, nil
}
// CanShare returns a boolean if this dependency is shareable.
func (d *CatalogNode) CanShare() bool {
func (d *CatalogNodeQuery) CanShare() bool {
return false
}
// HashCode returns a unique identifier.
func (d *CatalogNode) HashCode() string {
if d.dataCenter != "" {
return fmt.Sprintf("NodeDetail|%s@%s", d.rawKey, d.dataCenter)
// String returns the human-friendly version of this dependency.
func (d *CatalogNodeQuery) String() string {
name := d.name
if d.dc != "" {
name = name + "@" + d.dc
}
return fmt.Sprintf("NodeDetail|%s", d.rawKey)
}
// Display prints the human-friendly output.
func (d *CatalogNode) Display() string {
if d.dataCenter != "" {
return fmt.Sprintf("node(%s@%s)", d.rawKey, d.dataCenter)
if name == "" {
return "catalog.node"
}
return fmt.Sprintf(`"node(%s)"`, d.rawKey)
return fmt.Sprintf("catalog.node(%s)", name)
}
// Stop halts the dependency's fetch function.
func (d *CatalogNode) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
func (d *CatalogNodeQuery) Stop() {
close(d.stopCh)
}
// ParseCatalogNode parses a name name and optional datacenter value.
// If the name is empty or not provided then the current agent is used.
func ParseCatalogNode(s ...string) (*CatalogNode, error) {
switch len(s) {
case 0:
cn := &CatalogNode{stopCh: make(chan struct{})}
return cn, nil
case 1:
cn := &CatalogNode{
rawKey: s[0],
stopCh: make(chan struct{}),
}
return cn, nil
case 2:
dc := s[1]
re := regexp.MustCompile(`\A` +
`(@(?P<datacenter>[[:word:]\.\-]+))?` +
`\z`)
names := re.SubexpNames()
match := re.FindAllStringSubmatch(dc, -1)
if len(match) == 0 {
return nil, errors.New("invalid node dependency format")
}
r := match[0]
m := map[string]string{}
for i, n := range r {
if names[i] != "" {
m[names[i]] = n
}
}
nd := &CatalogNode{
rawKey: s[0],
dataCenter: m["datacenter"],
stopCh: make(chan struct{}),
}
return nd, nil
default:
return nil, fmt.Errorf("expected 0, 1, or 2 arguments, got %d", len(s))
}
// Type returns the type of this dependency.
func (d *CatalogNodeQuery) Type() Type {
return TypeConsul
}
// Sorting
// ByService is a sorter of node services by their service name and then ID.
type ByService []*CatalogNodeService
// NodeServiceList is a sortable list of node service names.
type NodeServiceList []*NodeService
func (s NodeServiceList) Len() int { return len(s) }
func (s NodeServiceList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s NodeServiceList) Less(i, j int) bool {
func (s ByService) Len() int { return len(s) }
func (s ByService) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ByService) Less(i, j int) bool {
if s[i].Service == s[j].Service {
return s[i].ID <= s[j].ID
}

View File

@@ -2,14 +2,21 @@ package dependency
import (
"encoding/gob"
"errors"
"fmt"
"log"
"net/url"
"regexp"
"sort"
"sync"
"github.com/hashicorp/consul/api"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*CatalogNodesQuery)(nil)
// CatalogNodesQueryRe is the regular expression to use.
CatalogNodesQueryRe = regexp.MustCompile(`\A` + dcRe + nearRe + `\z`)
)
func init() {
@@ -23,60 +30,53 @@ type Node struct {
TaggedAddresses map[string]string
}
// CatalogNodes is the representation of all registered nodes in Consul.
type CatalogNodes struct {
sync.Mutex
// CatalogNodesQuery is the representation of all registered nodes in Consul.
type CatalogNodesQuery struct {
stopCh chan struct{}
rawKey string
DataCenter string
stopped bool
stopCh chan struct{}
dc string
near string
}
// NewCatalogNodesQuery parses the given string into a dependency. If the name is
// empty then the name of the local agent is used.
func NewCatalogNodesQuery(s string) (*CatalogNodesQuery, error) {
if !CatalogNodesQueryRe.MatchString(s) {
return nil, fmt.Errorf("catalog.nodes: invalid format: %q", s)
}
m := regexpMatch(CatalogNodesQueryRe, s)
return &CatalogNodesQuery{
dc: m["dc"],
near: m["near"],
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Consul API defined by the given client and returns a slice
// of Node objects
func (d *CatalogNodes) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
consulOpts := opts.consulQueryOptions()
if d.DataCenter != "" {
consulOpts.Datacenter = d.DataCenter
}
consul, err := clients.Consul()
if err != nil {
return nil, nil, fmt.Errorf("catalog nodes: error getting client: %s", err)
}
var n []*api.Node
var qm *api.QueryMeta
dataCh := make(chan struct{})
go func() {
log.Printf("[DEBUG] (%s) querying Consul with %+v", d.Display(), consulOpts)
n, qm, err = consul.Catalog().Nodes(consulOpts)
close(dataCh)
}()
func (d *CatalogNodesQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-dataCh:
default:
}
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
Near: d.near,
})
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/catalog/nodes",
RawQuery: opts.String(),
})
n, qm, err := clients.Consul().Catalog().Nodes(opts.ToConsulOpts())
if err != nil {
return nil, nil, fmt.Errorf("catalog nodes: error fetching: %s", err)
return nil, nil, errors.Wrap(err, d.String())
}
log.Printf("[DEBUG] (%s) Consul returned %d nodes", d.Display(), len(n))
log.Printf("[TRACE] %s: returned %d results", d, len(n))
nodes := make([]*Node, 0, len(n))
for _, node := range n {
@@ -86,7 +86,7 @@ func (d *CatalogNodes) Fetch(clients *ClientSet, opts *QueryOptions) (interface{
TaggedAddresses: node.TaggedAddresses,
})
}
sort.Stable(NodeList(nodes))
sort.Stable(ByNode(nodes))
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
@@ -97,84 +97,42 @@ func (d *CatalogNodes) Fetch(clients *ClientSet, opts *QueryOptions) (interface{
}
// CanShare returns a boolean if this dependency is shareable.
func (d *CatalogNodes) CanShare() bool {
func (d *CatalogNodesQuery) CanShare() bool {
return true
}
// HashCode returns a unique identifier.
func (d *CatalogNodes) HashCode() string {
return fmt.Sprintf("CatalogNodes|%s", d.rawKey)
}
// Display prints the human-friendly output.
func (d *CatalogNodes) Display() string {
if d.rawKey == "" {
return fmt.Sprintf(`"nodes"`)
// String returns the human-friendly version of this dependency.
func (d *CatalogNodesQuery) String() string {
name := ""
if d.dc != "" {
name = name + "@" + d.dc
}
if d.near != "" {
name = name + "~" + d.near
}
return fmt.Sprintf(`"nodes(%s)"`, d.rawKey)
if name == "" {
return "catalog.nodes"
}
return fmt.Sprintf("catalog.nodes(%s)", name)
}
// Stop halts the dependency's fetch function.
func (d *CatalogNodes) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
func (d *CatalogNodesQuery) Stop() {
close(d.stopCh)
}
// ParseCatalogNodes parses a string of the format @dc.
func ParseCatalogNodes(s ...string) (*CatalogNodes, error) {
switch len(s) {
case 0:
cn := &CatalogNodes{
rawKey: "",
stopCh: make(chan struct{}),
}
return cn, nil
case 1:
dc := s[0]
re := regexp.MustCompile(`\A` +
`(@(?P<datacenter>[[:word:]\.\-]+))?` +
`\z`)
names := re.SubexpNames()
match := re.FindAllStringSubmatch(dc, -1)
if len(match) == 0 {
return nil, errors.New("invalid node dependency format")
}
r := match[0]
m := map[string]string{}
for i, n := range r {
if names[i] != "" {
m[names[i]] = n
}
}
cn := &CatalogNodes{
rawKey: dc,
DataCenter: m["datacenter"],
stopCh: make(chan struct{}),
}
return cn, nil
default:
return nil, fmt.Errorf("expected 0 or 1 arguments, got %d", len(s))
}
// Type returns the type of this dependency.
func (d *CatalogNodesQuery) Type() Type {
return TypeConsul
}
// NodeList is a sortable list of node objects by name and then IP address.
type NodeList []*Node
// ByNode is a sortable list of nodes by name and then IP address.
type ByNode []*Node
func (s NodeList) Len() int { return len(s) }
func (s NodeList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s NodeList) Less(i, j int) bool {
func (s ByNode) Len() int { return len(s) }
func (s ByNode) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ByNode) Less(i, j int) bool {
if s[i].Node == s[j].Node {
return s[i].Address <= s[j].Address
}

View File

@@ -0,0 +1,146 @@
package dependency
import (
"encoding/gob"
"fmt"
"log"
"net/url"
"regexp"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*CatalogServiceQuery)(nil)
// CatalogServiceQueryRe is the regular expression to use.
CatalogServiceQueryRe = regexp.MustCompile(`\A` + tagRe + nameRe + dcRe + nearRe + `\z`)
)
func init() {
gob.Register([]*CatalogSnippet{})
}
// CatalogService is a catalog entry in Consul.
type CatalogService struct {
Node string
Address string
TaggedAddresses map[string]string
ServiceID string
ServiceName string
ServiceAddress string
ServiceTags ServiceTags
ServicePort int
}
// CatalogServiceQuery is the representation of a requested catalog services
// dependency from inside a template.
type CatalogServiceQuery struct {
stopCh chan struct{}
dc string
name string
near string
tag string
}
// NewCatalogServiceQuery parses a string into a CatalogServiceQuery.
func NewCatalogServiceQuery(s string) (*CatalogServiceQuery, error) {
if !CatalogServiceQueryRe.MatchString(s) {
return nil, fmt.Errorf("catalog.service: invalid format: %q", s)
}
m := regexpMatch(CatalogServiceQueryRe, s)
return &CatalogServiceQuery{
stopCh: make(chan struct{}, 1),
dc: m["dc"],
name: m["name"],
near: m["near"],
tag: m["tag"],
}, nil
}
// Fetch queries the Consul API defined by the given client and returns a slice
// of CatalogService objects.
func (d *CatalogServiceQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
default:
}
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
Near: d.near,
})
u := &url.URL{
Path: "/v1/catalog/service/" + d.name,
RawQuery: opts.String(),
}
if d.tag != "" {
q := u.Query()
q.Set("tag", d.tag)
u.RawQuery = q.Encode()
}
log.Printf("[TRACE] %s: GET %s", d, u)
entries, qm, err := clients.Consul().Catalog().Service(d.name, d.tag, opts.ToConsulOpts())
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
log.Printf("[TRACE] %s: returned %d results", d, len(entries))
var list []*CatalogService
for _, s := range entries {
list = append(list, &CatalogService{
Node: s.Node,
Address: s.Address,
TaggedAddresses: s.TaggedAddresses,
ServiceID: s.ServiceID,
ServiceName: s.ServiceName,
ServiceAddress: s.ServiceAddress,
ServiceTags: ServiceTags(deepCopyAndSortTags(s.ServiceTags)),
ServicePort: s.ServicePort,
})
}
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
LastContact: qm.LastContact,
}
return list, rm, nil
}
// CanShare returns a boolean if this dependency is shareable.
func (d *CatalogServiceQuery) CanShare() bool {
return true
}
// String returns the human-friendly version of this dependency.
func (d *CatalogServiceQuery) String() string {
name := d.name
if d.tag != "" {
name = d.tag + "." + name
}
if d.dc != "" {
name = name + "@" + d.dc
}
if d.near != "" {
name = name + "~" + d.near
}
return fmt.Sprintf("catalog.service(%s)", name)
}
// Stop halts the dependency's fetch function.
func (d *CatalogServiceQuery) Stop() {
close(d.stopCh)
}
// Type returns the type of this dependency.
func (d *CatalogServiceQuery) Type() Type {
return TypeConsul
}

View File

@@ -2,94 +2,87 @@ package dependency
import (
"encoding/gob"
"errors"
"fmt"
"log"
"net/url"
"regexp"
"sort"
"sync"
"github.com/hashicorp/consul/api"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*CatalogServicesQuery)(nil)
// CatalogServicesQueryRe is the regular expression to use for CatalogNodesQuery.
CatalogServicesQueryRe = regexp.MustCompile(`\A` + dcRe + `\z`)
)
func init() {
gob.Register([]*CatalogService{})
gob.Register([]*CatalogSnippet{})
}
// CatalogService is a catalog entry in Consul.
type CatalogService struct {
// CatalogSnippet is a catalog entry in Consul.
type CatalogSnippet struct {
Name string
Tags ServiceTags
}
// CatalogServices is the representation of a requested catalog service
// CatalogServicesQuery is the representation of a requested catalog service
// dependency from inside a template.
type CatalogServices struct {
sync.Mutex
type CatalogServicesQuery struct {
stopCh chan struct{}
rawKey string
Name string
Tags []string
DataCenter string
stopped bool
stopCh chan struct{}
dc string
}
// NewCatalogServicesQuery parses a string of the format @dc.
func NewCatalogServicesQuery(s string) (*CatalogServicesQuery, error) {
if !CatalogServicesQueryRe.MatchString(s) {
return nil, fmt.Errorf("catalog.services: invalid format: %q", s)
}
m := regexpMatch(CatalogServicesQueryRe, s)
return &CatalogServicesQuery{
dc: m["dc"],
}, nil
}
// Fetch queries the Consul API defined by the given client and returns a slice
// of CatalogService objects.
func (d *CatalogServices) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
consulOpts := opts.consulQueryOptions()
if d.DataCenter != "" {
consulOpts.Datacenter = d.DataCenter
}
consul, err := clients.Consul()
if err != nil {
return nil, nil, fmt.Errorf("catalog services: error getting client: %s", err)
}
var entries map[string][]string
var qm *api.QueryMeta
dataCh := make(chan struct{})
go func() {
log.Printf("[DEBUG] (%s) querying Consul with %+v", d.Display(), consulOpts)
entries, qm, err = consul.Catalog().Services(consulOpts)
close(dataCh)
}()
func (d *CatalogServicesQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-dataCh:
default:
}
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
})
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/catalog/services",
RawQuery: opts.String(),
})
entries, qm, err := clients.Consul().Catalog().Services(opts.ToConsulOpts())
if err != nil {
return nil, nil, fmt.Errorf("catalog services: error fetching: %s", err)
return nil, nil, errors.Wrap(err, d.String())
}
log.Printf("[DEBUG] (%s) Consul returned %d catalog services", d.Display(), len(entries))
log.Printf("[TRACE] %s: returned %d results", d, len(entries))
var catalogServices []*CatalogService
var catalogServices []*CatalogSnippet
for name, tags := range entries {
tags = deepCopyAndSortTags(tags)
catalogServices = append(catalogServices, &CatalogService{
catalogServices = append(catalogServices, &CatalogSnippet{
Name: name,
Tags: ServiceTags(tags),
Tags: ServiceTags(deepCopyAndSortTags(tags)),
})
}
sort.Stable(CatalogServicesList(catalogServices))
sort.Stable(ByName(catalogServices))
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
@@ -100,86 +93,34 @@ func (d *CatalogServices) Fetch(clients *ClientSet, opts *QueryOptions) (interfa
}
// CanShare returns a boolean if this dependency is shareable.
func (d *CatalogServices) CanShare() bool {
func (d *CatalogServicesQuery) CanShare() bool {
return true
}
// HashCode returns a unique identifier.
func (d *CatalogServices) HashCode() string {
return fmt.Sprintf("CatalogServices|%s", d.rawKey)
}
// Display prints the human-friendly output.
func (d *CatalogServices) Display() string {
if d.rawKey == "" {
return fmt.Sprintf(`"services"`)
// String returns the human-friendly version of this dependency.
func (d *CatalogServicesQuery) String() string {
if d.dc != "" {
return fmt.Sprintf("catalog.services(@%s)", d.dc)
}
return fmt.Sprintf(`"services(%s)"`, d.rawKey)
return "catalog.services"
}
// Stop halts the dependency's fetch function.
func (d *CatalogServices) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
func (d *CatalogServicesQuery) Stop() {
close(d.stopCh)
}
// ParseCatalogServices parses a string of the format @dc.
func ParseCatalogServices(s ...string) (*CatalogServices, error) {
switch len(s) {
case 0:
cs := &CatalogServices{
rawKey: "",
stopCh: make(chan struct{}),
}
return cs, nil
case 1:
dc := s[0]
re := regexp.MustCompile(`\A` +
`(@(?P<datacenter>[[:word:]\.\-]+))?` +
`\z`)
names := re.SubexpNames()
match := re.FindAllStringSubmatch(dc, -1)
if len(match) == 0 {
return nil, errors.New("invalid catalog service dependency format")
}
r := match[0]
m := map[string]string{}
for i, n := range r {
if names[i] != "" {
m[names[i]] = n
}
}
nd := &CatalogServices{
rawKey: dc,
DataCenter: m["datacenter"],
stopCh: make(chan struct{}),
}
return nd, nil
default:
return nil, fmt.Errorf("expected 0 or 1 arguments, got %d", len(s))
}
// Type returns the type of this dependency.
func (d *CatalogServicesQuery) Type() Type {
return TypeConsul
}
/// --- Sorting
// ByName is a sortable slice of CatalogService structs.
type ByName []*CatalogSnippet
// CatalogServicesList is a sortable slice of CatalogService structs.
type CatalogServicesList []*CatalogService
func (s CatalogServicesList) Len() int { return len(s) }
func (s CatalogServicesList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s CatalogServicesList) Less(i, j int) bool {
func (s ByName) Len() int { return len(s) }
func (s ByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ByName) Less(i, j int) bool {
if s[i].Name <= s[j].Name {
return true
}

View File

@@ -71,26 +71,17 @@ func NewClientSet() *ClientSet {
// CreateConsulClient creates a new Consul API client from the given input.
func (c *ClientSet) CreateConsulClient(i *CreateConsulClientInput) error {
log.Printf("[INFO] (clients) creating consul/api client")
// Generate the default config
consulConfig := consulapi.DefaultConfig()
// Set the address
if i.Address != "" {
log.Printf("[DEBUG] (clients) setting consul address to %q", i.Address)
consulConfig.Address = i.Address
}
// Configure the token
if i.Token != "" {
log.Printf("[DEBUG] (clients) setting consul token")
consulConfig.Token = i.Token
}
// Add basic auth
if i.AuthEnabled {
log.Printf("[DEBUG] (clients) setting basic auth")
consulConfig.HttpAuth = &consulapi.HttpBasicAuth{
Username: i.AuthUsername,
Password: i.AuthPassword,
@@ -102,7 +93,6 @@ func (c *ClientSet) CreateConsulClient(i *CreateConsulClientInput) error {
// Configure SSL
if i.SSLEnabled {
log.Printf("[DEBUG] (clients) enabling consul SSL")
consulConfig.Scheme = "https"
var tlsConfig tls.Config
@@ -140,7 +130,6 @@ func (c *ClientSet) CreateConsulClient(i *CreateConsulClientInput) error {
if i.ServerName != "" {
tlsConfig.ServerName = i.ServerName
tlsConfig.InsecureSkipVerify = false
log.Printf("[DEBUG] (clients) using explicit consul TLS server host name: %s", tlsConfig.ServerName)
}
if !i.SSLVerify {
log.Printf("[WARN] (clients) disabling consul SSL verification")
@@ -161,23 +150,20 @@ func (c *ClientSet) CreateConsulClient(i *CreateConsulClientInput) error {
}
// Save the data on ourselves
c.Lock()
c.consul = &consulClient{
client: client,
httpClient: consulConfig.HttpClient,
}
c.Unlock()
return nil
}
func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error {
log.Printf("[INFO] (clients) creating vault/api client")
// Generate the default config
vaultConfig := vaultapi.DefaultConfig()
// Set the address
if i.Address != "" {
log.Printf("[DEBUG] (clients) setting vault address to %q", i.Address)
vaultConfig.Address = i.Address
}
@@ -186,7 +172,6 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error {
// Configure SSL
if i.SSLEnabled {
log.Printf("[DEBUG] (clients) enabling vault SSL")
var tlsConfig tls.Config
// Custom certificate or certificate and key
@@ -222,7 +207,6 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error {
if i.ServerName != "" {
tlsConfig.ServerName = i.ServerName
tlsConfig.InsecureSkipVerify = false
log.Printf("[DEBUG] (clients) using explicit vault TLS server host name: %s", tlsConfig.ServerName)
}
if !i.SSLVerify {
log.Printf("[WARN] (clients) disabling vault SSL verification")
@@ -244,13 +228,11 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error {
// Set the token if given
if i.Token != "" {
log.Printf("[DEBUG] (clients) setting vault token")
client.SetToken(i.Token)
}
// Check if we are unwrapping
if i.UnwrapToken {
log.Printf("[INFO] (clients) unwrapping vault token")
secret, err := client.Logical().Unwrap(i.Token)
if err != nil {
return fmt.Errorf("client set: vault unwrap: %s", err)
@@ -272,40 +254,28 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error {
}
// Save the data on ourselves
c.Lock()
c.vault = &vaultClient{
client: client,
httpClient: vaultConfig.HttpClient,
}
c.Unlock()
return nil
}
// Consul returns the Consul client for this clientset, or an error if no
// Consul client has been set.
func (c *ClientSet) Consul() (*consulapi.Client, error) {
// Consul returns the Consul client for this set.
func (c *ClientSet) Consul() *consulapi.Client {
c.RLock()
defer c.RUnlock()
if c.consul == nil {
return nil, fmt.Errorf("clientset: missing consul client")
}
cp := new(consulapi.Client)
*cp = *c.consul.client
return cp, nil
return c.consul.client
}
// Vault returns the Vault client for this clientset, or an error if no
// Vault client has been set.
func (c *ClientSet) Vault() (*vaultapi.Client, error) {
// Vault returns the Consul client for this set.
func (c *ClientSet) Vault() *vaultapi.Client {
c.RLock()
defer c.RUnlock()
if c.vault == nil {
return nil, fmt.Errorf("clientset: missing vault client")
}
cp := new(vaultapi.Client)
*cp = *c.vault.client
return cp, nil
return c.vault.client
}
// Stop closes all idle connections for any attached clients.

View File

@@ -1,118 +0,0 @@
package dependency
import (
"fmt"
"log"
"sort"
"sync"
"time"
)
var sleepTime = 15 * time.Second
// Datacenters is the dependency to query all datacenters
type Datacenters struct {
sync.Mutex
rawKey string
stopped bool
stopCh chan struct{}
}
// Fetch queries the Consul API defined by the given client and returns a slice
// of strings representing the datacenters
func (d *Datacenters) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
log.Printf("[DEBUG] (%s) querying Consul with %+v", d.Display(), opts)
// This is pretty ghetto, but the datacenters endpoint does not support
// blocking queries, so we are going to "fake it until we make it". When we
// first query, the LastIndex will be "0", meaning we should immediately
// return data, but future calls will include a LastIndex. If we have a
// LastIndex in the query metadata, sleep for 15 seconds before asking Consul
// again.
//
// This is probably okay given the frequency in which datacenters actually
// change, but is technically not edge-triggering.
if opts.WaitIndex != 0 {
log.Printf("[DEBUG] (%s) pretending to long-poll", d.Display())
select {
case <-d.stopCh:
log.Printf("[DEBUG] (%s) received interrupt", d.Display())
return nil, nil, ErrStopped
case <-time.After(sleepTime):
}
}
consul, err := clients.Consul()
if err != nil {
return nil, nil, fmt.Errorf("datacenters: error getting client: %s", err)
}
catalog := consul.Catalog()
result, err := catalog.Datacenters()
if err != nil {
return nil, nil, fmt.Errorf("datacenters: error fetching: %s", err)
}
log.Printf("[DEBUG] (%s) Consul returned %d datacenters", d.Display(), len(result))
sort.Strings(result)
return respWithMetadata(result)
}
// CanShare returns if this dependency is shareable.
func (d *Datacenters) CanShare() bool {
return true
}
// HashCode returns the hash code for this dependency.
func (d *Datacenters) HashCode() string {
return fmt.Sprintf("Datacenters|%s", d.rawKey)
}
// Display returns a string that should be displayed to the user in output (for
// example).
func (d *Datacenters) Display() string {
if d.rawKey == "" {
return fmt.Sprintf(`"datacenters"`)
}
return fmt.Sprintf(`"datacenters(%s)"`, d.rawKey)
}
// Stop terminates this dependency's execution early.
func (d *Datacenters) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
}
// ParseDatacenters creates a new datacenter dependency.
func ParseDatacenters(s ...string) (*Datacenters, error) {
switch len(s) {
case 0:
dcs := &Datacenters{
rawKey: "",
stopCh: make(chan struct{}, 0),
}
return dcs, nil
default:
return nil, fmt.Errorf("expected 0 arguments, got %d", len(s))
}
}

View File

@@ -1,66 +1,52 @@
package dependency
import (
"errors"
"fmt"
"log"
"net/url"
"regexp"
"sort"
"strconv"
"time"
consulapi "github.com/hashicorp/consul/api"
)
// ErrStopped is a special error that is returned when a dependency is
// prematurely stopped, usually due to a configuration reload or a process
// interrupt.
var ErrStopped = errors.New("dependency stopped")
const (
dcRe = `(@(?P<dc>[[:word:]\.\-\_]+))?`
keyRe = `/?(?P<key>[^@]+)`
filterRe = `(\|(?P<filter>[[:word:]\,]+))?`
nameRe = `(?P<name>[[:word:]\-\_]+)`
nearRe = `(~(?P<near>[[:word:]\.\-\_]+))?`
prefixRe = `/?(?P<prefix>[^@]+)`
tagRe = `((?P<tag>[[:word:]\.\-\_]+)\.)?`
)
type Type int
const (
TypeConsul Type = iota
TypeVault
TypeLocal
)
// Dependency is an interface for a dependency that Consul Template is capable
// of watching.
type Dependency interface {
Fetch(*ClientSet, *QueryOptions) (interface{}, *ResponseMetadata, error)
CanShare() bool
HashCode() string
Display() string
String() string
Stop()
}
// FetchError is a special kind of error returned by the Fetch method that
// contains additional metadata which informs the caller how to respond. This
// error implements the standard Error interface, so it can be passed as a
// regular error down the stack.
type FetchError struct {
originalError error
shouldExit bool
}
func (e *FetchError) Error() string {
return e.originalError.Error()
}
func (e *FetchError) OriginalError() error {
return e.originalError
}
func (e *FetchError) ShouldExit() bool {
return e.shouldExit
}
func ErrWithExit(err error) *FetchError {
return &FetchError{
originalError: err,
shouldExit: true,
}
}
func ErrWithExitf(s string, i ...interface{}) *FetchError {
return ErrWithExit(fmt.Errorf(s, i...))
Type() Type
}
// ServiceTags is a slice of tags assigned to a Service
type ServiceTags []string
// Contains returns true if the tags exists in the ServiceTags slice.
// This is deprecated and should not be used.
func (t ServiceTags) Contains(s string) bool {
log.Printf("[WARN] .Tags.Contains is deprecated. Use the built-in\n" +
"functions 'in' or 'contains' with a pipe instead.")
for _, v := range t {
if v == s {
return true
@@ -73,18 +59,97 @@ func (t ServiceTags) Contains(s string) bool {
// client-agnostic, and the dependency determines which, if any, of the options
// to use.
type QueryOptions struct {
AllowStale bool
WaitIndex uint64
WaitTime time.Duration
AllowStale bool
Datacenter string
Near string
RequireConsistent bool
WaitIndex uint64
WaitTime time.Duration
}
// Converts the query options to Consul API ready query options.
func (r *QueryOptions) consulQueryOptions() *consulapi.QueryOptions {
return &consulapi.QueryOptions{
AllowStale: r.AllowStale,
WaitIndex: r.WaitIndex,
WaitTime: r.WaitTime,
func (q *QueryOptions) Merge(o *QueryOptions) *QueryOptions {
var r QueryOptions
if q == nil {
if o == nil {
return &QueryOptions{}
}
r = *o
return &r
}
r = *q
if o == nil {
return &r
}
if o.AllowStale != false {
r.AllowStale = o.AllowStale
}
if o.Datacenter != "" {
r.Datacenter = o.Datacenter
}
if o.Near != "" {
r.Near = o.Near
}
if o.RequireConsistent != false {
r.RequireConsistent = o.RequireConsistent
}
if o.WaitIndex != 0 {
r.WaitIndex = o.WaitIndex
}
if o.WaitTime != 0 {
r.WaitTime = o.WaitTime
}
return &r
}
func (q *QueryOptions) ToConsulOpts() *consulapi.QueryOptions {
return &consulapi.QueryOptions{
AllowStale: q.AllowStale,
Datacenter: q.Datacenter,
Near: q.Near,
RequireConsistent: q.RequireConsistent,
WaitIndex: q.WaitIndex,
WaitTime: q.WaitTime,
}
}
func (q *QueryOptions) String() string {
u := &url.Values{}
if q.AllowStale {
u.Add("stale", strconv.FormatBool(q.AllowStale))
}
if q.Datacenter != "" {
u.Add("dc", q.Datacenter)
}
if q.Near != "" {
u.Add("near", q.Near)
}
if q.RequireConsistent {
u.Add("consistent", strconv.FormatBool(q.RequireConsistent))
}
if q.WaitIndex != 0 {
u.Add("index", strconv.FormatUint(q.WaitIndex, 10))
}
if q.WaitTime != 0 {
u.Add("wait", q.WaitTime.String())
}
return u.Encode()
}
// ResponseMetadata is a struct that contains metadata about the response. This
@@ -92,6 +157,7 @@ func (r *QueryOptions) consulQueryOptions() *consulapi.QueryOptions {
type ResponseMetadata struct {
LastIndex uint64
LastContact time.Duration
Block bool
}
// deepCopyAndSortTags deep copies the tags in the given string slice and then
@@ -113,3 +179,23 @@ func respWithMetadata(i interface{}) (interface{}, *ResponseMetadata, error) {
LastIndex: uint64(time.Now().Unix()),
}, nil
}
// regexpMatch matches the given regexp and extracts the match groups into a
// named map.
func regexpMatch(re *regexp.Regexp, q string) map[string]string {
names := re.SubexpNames()
match := re.FindAllStringSubmatch(q, -1)
if len(match) == 0 {
return map[string]string{}
}
m := map[string]string{}
for i, n := range match[0] {
if names[i] != "" {
m[names[i]] = n
}
}
return m
}

View File

@@ -0,0 +1,84 @@
package dependency
import (
"fmt"
"log"
"os"
"strings"
"time"
)
var (
// Ensure implements
_ Dependency = (*EnvQuery)(nil)
// EnvQuerySleepTime is the amount of time to sleep between queries. Since
// it's not supporting to change a running processes' environment, this can
// be a fairly large value.
EnvQuerySleepTime = 5 * time.Minute
)
// EnvQuery represents a local file dependency.
type EnvQuery struct {
stopCh chan struct{}
key string
stat os.FileInfo
}
// NewEnvQuery creates a file dependency from the given key.
func NewEnvQuery(s string) (*EnvQuery, error) {
s = strings.TrimSpace(s)
if s == "" {
return nil, fmt.Errorf("env: invalid format: %q", s)
}
return &EnvQuery{
key: s,
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch retrieves this dependency and returns the result or any errors that
// occur in the process.
func (d *EnvQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
opts = opts.Merge(&QueryOptions{})
log.Printf("[TRACE] %s: ENV %s", d, d.key)
if opts.WaitIndex != 0 {
log.Printf("[TRACE] %s: long polling for %s", d, EnvQuerySleepTime)
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-time.After(EnvQuerySleepTime):
}
}
result := os.Getenv(d.key)
log.Printf("[TRACE] %s: returned result", d)
return respWithMetadata(result)
}
// CanShare returns a boolean if this dependency is shareable.
func (d *EnvQuery) CanShare() bool {
return false
}
// Stop halts the dependency's fetch function.
func (d *EnvQuery) Stop() {
close(d.stopCh)
}
// String returns the human-friendly version of this dependency.
func (d *EnvQuery) String() string {
return fmt.Sprintf("env(%s)", d.key)
}
// Type returns the type of this dependency.
func (d *EnvQuery) Type() Type {
return TypeLocal
}

View File

@@ -0,0 +1,11 @@
package dependency
import "errors"
// ErrStopped is a special error that is returned when a dependency is
// prematurely stopped, usually due to a configuration reload or a process
// interrupt.
var ErrStopped = errors.New("dependency stopped")
// ErrContinue is a special error which says to continue (retry) on error.
var ErrContinue = errors.New("dependency continue")

View File

@@ -1,135 +1,129 @@
package dependency
import (
"errors"
"fmt"
"io/ioutil"
"log"
"os"
"sync"
"strings"
"time"
"github.com/pkg/errors"
)
// File represents a local file dependency.
type File struct {
sync.Mutex
mutex sync.RWMutex
rawKey string
lastStat os.FileInfo
stopped bool
stopCh chan struct{}
var (
// Ensure implements
_ Dependency = (*FileQuery)(nil)
// FileQuerySleepTime is the amount of time to sleep between queries, since
// the fsnotify library is not compatible with solaris and other OSes yet.
FileQuerySleepTime = 2 * time.Second
)
// FileQuery represents a local file dependency.
type FileQuery struct {
stopCh chan struct{}
path string
stat os.FileInfo
}
// NewFileQuery creates a file dependency from the given path.
func NewFileQuery(s string) (*FileQuery, error) {
s = strings.TrimSpace(s)
if s == "" {
return nil, fmt.Errorf("file: invalid format: %q", s)
}
return &FileQuery{
path: s,
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch retrieves this dependency and returns the result or any errors that
// occur in the process.
func (d *File) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
var err error
var newStat os.FileInfo
var data []byte
dataCh := make(chan struct{})
go func() {
log.Printf("[DEBUG] (%s) querying file", d.Display())
newStat, err = d.watch()
close(dataCh)
}()
func (d *FileQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
log.Printf("[TRACE] %s: READ %s", d, d.path)
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-dataCh:
}
log.Printf("[TRACE] %s: stopped", d)
return "", nil, ErrStopped
case r := <-d.watch(d.stat):
if r.err != nil {
return "", nil, errors.Wrap(r.err, d.String())
}
if err != nil {
return nil, nil, fmt.Errorf("file: error watching: %s", err)
}
log.Printf("[TRACE] %s: reported change", d)
d.mutex.Lock()
defer d.mutex.Unlock()
d.lastStat = newStat
data, err := ioutil.ReadFile(d.path)
if err != nil {
return "", nil, errors.Wrap(err, d.String())
}
if data, err = ioutil.ReadFile(d.rawKey); err == nil {
d.stat = r.stat
return respWithMetadata(string(data))
}
return nil, nil, fmt.Errorf("file: error reading: %s", err)
}
// CanShare returns a boolean if this dependency is shareable.
func (d *File) CanShare() bool {
func (d *FileQuery) CanShare() bool {
return false
}
// HashCode returns a unique identifier.
func (d *File) HashCode() string {
return fmt.Sprintf("StoreKeyPrefix|%s", d.rawKey)
}
// Display prints the human-friendly output.
func (d *File) Display() string {
return fmt.Sprintf(`"file(%s)"`, d.rawKey)
}
// Stop halts the dependency's fetch function.
func (d *File) Stop() {
d.Lock()
defer d.Unlock()
func (d *FileQuery) Stop() {
close(d.stopCh)
}
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
// String returns the human-friendly version of this dependency.
func (d *FileQuery) String() string {
return fmt.Sprintf("file(%s)", d.path)
}
// Type returns the type of this dependency.
func (d *FileQuery) Type() Type {
return TypeLocal
}
type watchResult struct {
stat os.FileInfo
err error
}
// watch watchers the file for changes
func (d *File) watch() (os.FileInfo, error) {
for {
stat, err := os.Stat(d.rawKey)
if err != nil {
return nil, err
func (d *FileQuery) watch(lastStat os.FileInfo) <-chan *watchResult {
ch := make(chan *watchResult, 1)
go func(lastStat os.FileInfo) {
for {
stat, err := os.Stat(d.path)
if err != nil {
select {
case <-d.stopCh:
return
case ch <- &watchResult{err: err}:
return
}
}
changed := lastStat == nil ||
lastStat.Size() != stat.Size() ||
lastStat.ModTime() != stat.ModTime()
if changed {
select {
case <-d.stopCh:
return
case ch <- &watchResult{stat: stat}:
return
}
}
time.Sleep(FileQuerySleepTime)
}
}(lastStat)
changed := func(d *File, stat os.FileInfo) bool {
d.mutex.RLock()
defer d.mutex.RUnlock()
if d.lastStat == nil {
return true
}
if d.lastStat.Size() != stat.Size() {
return true
}
if d.lastStat.ModTime() != stat.ModTime() {
return true
}
return false
}(d, stat)
if changed {
return stat, nil
}
time.Sleep(3 * time.Second)
}
}
// ParseFile creates a file dependency from the given path.
func ParseFile(s string) (*File, error) {
if len(s) == 0 {
return nil, errors.New("cannot specify empty file dependency")
}
kd := &File{
rawKey: s,
stopCh: make(chan struct{}),
}
return kd, nil
return ch
}

View File

@@ -2,27 +2,21 @@ package dependency
import (
"encoding/gob"
"errors"
"fmt"
"log"
"net/url"
"regexp"
"sort"
"strings"
"sync"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
)
func init() {
gob.Register([]*HealthService{})
}
const (
HealthAny = "any"
HealthPassing = "passing"
HealthWarning = "warning"
HealthUnknown = "unknown"
HealthCritical = "critical"
HealthMaint = "maintenance"
@@ -30,6 +24,18 @@ const (
ServiceMaint = "_service_maintenance:"
)
var (
// Ensure implements
_ Dependency = (*HealthServiceQuery)(nil)
// HealthServiceQueryRe is the regular expression to use.
HealthServiceQueryRe = regexp.MustCompile(`\A` + tagRe + nameRe + dcRe + nearRe + filterRe + `\z`)
)
func init() {
gob.Register([]*HealthService{})
}
// HealthService is a service entry in Consul.
type HealthService struct {
Node string
@@ -40,371 +46,188 @@ type HealthService struct {
Tags ServiceTags
Checks []*api.HealthCheck
Status string
Port uint64
Port int
}
// HealthServices is the struct that is formed from the dependency inside a
// template.
type HealthServices struct {
sync.Mutex
// HealthServiceQuery is the representation of all a service query in Consul.
type HealthServiceQuery struct {
stopCh chan struct{}
rawKey string
Name string
Tag string
DataCenter string
StatusFilter ServiceStatusFilter
stopped bool
stopCh chan struct{}
dc string
filters []string
name string
near string
tag string
}
// NewHealthServiceQuery processes the strings to build a service dependency.
func NewHealthServiceQuery(s string) (*HealthServiceQuery, error) {
if !HealthServiceQueryRe.MatchString(s) {
return nil, fmt.Errorf("health.service: invalid format: %q", s)
}
m := regexpMatch(HealthServiceQueryRe, s)
var filters []string
if filter := m["filter"]; filter != "" {
split := strings.Split(filter, ",")
for _, f := range split {
f = strings.TrimSpace(f)
switch f {
case HealthAny,
HealthPassing,
HealthWarning,
HealthCritical,
HealthMaint:
filters = append(filters, f)
case "":
default:
return nil, fmt.Errorf("health.service: invalid filter: %q in %q", f, s)
}
}
sort.Strings(filters)
} else {
filters = []string{HealthPassing}
}
return &HealthServiceQuery{
stopCh: make(chan struct{}, 1),
dc: m["dc"],
filters: filters,
name: m["name"],
near: m["near"],
tag: m["tag"],
}, nil
}
// Fetch queries the Consul API defined by the given client and returns a slice
// of HealthService objects.
func (d *HealthServices) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
consulOpts := opts.consulQueryOptions()
if d.DataCenter != "" {
consulOpts.Datacenter = d.DataCenter
}
onlyHealthy := false
if d.StatusFilter == nil {
onlyHealthy = true
}
consul, err := clients.Consul()
if err != nil {
return nil, nil, fmt.Errorf("health services: error getting client: %s", err)
}
var entries []*api.ServiceEntry
var qm *api.QueryMeta
dataCh := make(chan struct{})
go func() {
log.Printf("[DEBUG] (%s) querying consul with %+v", d.Display(), consulOpts)
entries, qm, err = consul.Health().Service(d.Name, d.Tag, onlyHealthy, consulOpts)
close(dataCh)
}()
func (d *HealthServiceQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-dataCh:
default:
}
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
Near: d.near,
})
u := &url.URL{
Path: "/v1/health/service/" + d.name,
RawQuery: opts.String(),
}
if d.tag != "" {
q := u.Query()
q.Set("tag", d.tag)
u.RawQuery = q.Encode()
}
log.Printf("[TRACE] %s: GET %s", d, u)
// Check if a user-supplied filter was given. If so, we may be querying for
// more than healthy services, so we need to implement client-side filtering.
passingOnly := len(d.filters) == 1 && d.filters[0] == HealthPassing
entries, qm, err := clients.Consul().Health().Service(d.name, d.tag, passingOnly, opts.ToConsulOpts())
if err != nil {
return nil, nil, fmt.Errorf("health services: error fetching: %s", err)
return nil, nil, errors.Wrap(err, d.String())
}
log.Printf("[DEBUG] (%s) Consul returned %d services", d.Display(), len(entries))
services := make([]*HealthService, 0, len(entries))
log.Printf("[TRACE] %s: returned %d results", d, len(entries))
list := make([]*HealthService, 0, len(entries))
for _, entry := range entries {
// Get the status of this service from its checks.
status, err := statusFromChecks(entry.Checks)
if err != nil {
return nil, nil, fmt.Errorf("health services: "+
"error getting status from checks: %s", err)
}
status := entry.Checks.AggregatedStatus()
// If we are not checking only healthy services, filter out services that do
// not match the given filter.
if d.StatusFilter != nil && !d.StatusFilter.Accept(status) {
if !acceptStatus(d.filters, status) {
continue
}
// Sort the tags.
tags := deepCopyAndSortTags(entry.Service.Tags)
// Get the address of the service, falling back to the address of the node.
var address string
if entry.Service.Address != "" {
address = entry.Service.Address
} else {
address := entry.Service.Address
if address == "" {
address = entry.Node.Address
}
services = append(services, &HealthService{
list = append(list, &HealthService{
Node: entry.Node.Node,
NodeAddress: entry.Node.Address,
Address: address,
ID: entry.Service.ID,
Name: entry.Service.Service,
Tags: tags,
Tags: ServiceTags(deepCopyAndSortTags(entry.Service.Tags)),
Status: status,
Checks: entry.Checks,
Port: uint64(entry.Service.Port),
Port: entry.Service.Port,
})
}
log.Printf("[DEBUG] (%s) %d services after health check status filtering", d.Display(), len(services))
log.Printf("[TRACE] %s: returned %d results after filtering", d, len(list))
sort.Stable(HealthServiceList(services))
sort.Stable(ByNodeThenID(list))
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
LastContact: qm.LastContact,
}
return services, rm, nil
return list, rm, nil
}
// CanShare returns a boolean if this dependency is shareable.
func (d *HealthServices) CanShare() bool {
func (d *HealthServiceQuery) CanShare() bool {
return true
}
// HashCode returns a unique identifier.
func (d *HealthServices) HashCode() string {
return fmt.Sprintf("HealthServices|%s", d.rawKey)
}
// Display prints the human-friendly output.
func (d *HealthServices) Display() string {
return fmt.Sprintf(`"service(%s)"`, d.rawKey)
}
// Stop halts the dependency's fetch function.
func (d *HealthServices) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
func (d *HealthServiceQuery) Stop() {
close(d.stopCh)
}
// ParseHealthServices processes the incoming strings to build a service dependency.
//
// Supported arguments
// ParseHealthServices("service_id")
// ParseHealthServices("service_id", "health_check")
//
// Where service_id is in the format of service(.tag(@datacenter))
// and health_check is either "any" or "passing".
//
// If no health_check is provided then its the same as "passing".
func ParseHealthServices(s ...string) (*HealthServices, error) {
var query string
var filter ServiceStatusFilter
var err error
switch len(s) {
case 1:
query = s[0]
filter, err = NewServiceStatusFilter("")
if err != nil {
return nil, err
}
case 2:
query = s[0]
filter, err = NewServiceStatusFilter(s[1])
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("expected 1 or 2 arguments, got %d", len(s))
// String returns the human-friendly version of this dependency.
func (d *HealthServiceQuery) String() string {
name := d.name
if d.tag != "" {
name = d.tag + "." + name
}
if len(query) == 0 {
return nil, errors.New("cannot specify empty health service dependency")
if d.dc != "" {
name = name + "@" + d.dc
}
re := regexp.MustCompile(`\A` +
`((?P<tag>[[:word:]\-.]+)\.)?` +
`((?P<name>[[:word:]\-/_]+))` +
`(@(?P<datacenter>[[:word:]\.\-]+))?(:(?P<port>[0-9]+))?` +
`\z`)
names := re.SubexpNames()
match := re.FindAllStringSubmatch(query, -1)
if len(match) == 0 {
return nil, errors.New("invalid health service dependency format")
if d.near != "" {
name = name + "~" + d.near
}
r := match[0]
m := map[string]string{}
for i, n := range r {
if names[i] != "" {
m[names[i]] = n
}
if len(d.filters) > 0 {
name = name + "|" + strings.Join(d.filters, ",")
}
tag, name, datacenter, port := m["tag"], m["name"], m["datacenter"], m["port"]
if name == "" {
return nil, errors.New("name part is required")
}
if port != "" {
log.Printf("[WARN] specifying a port in a 'service' query is not "+
"supported - please remove the port from the query %q", query)
}
var key string
if filter == nil {
key = query
} else {
key = fmt.Sprintf("%s %s", query, filter)
}
sd := &HealthServices{
rawKey: key,
Name: name,
Tag: tag,
DataCenter: datacenter,
StatusFilter: filter,
stopCh: make(chan struct{}),
}
return sd, nil
return fmt.Sprintf("health.service(%s)", name)
}
// statusFromChecks accepts a list of checks and returns the most likely status
// given those checks. Any "critical" statuses will automatically mark the
// service as critical. After that, any "unknown" statuses will mark as
// "unknown". If any warning checks exist, the status will be marked as
// "warning", and finally "passing". If there are no checks, the service will be
// marked as "passing".
func statusFromChecks(checks []*api.HealthCheck) (string, error) {
var passing, warning, unknown, critical, maintenance bool
for _, check := range checks {
if check.CheckID == NodeMaint || strings.HasPrefix(check.CheckID, ServiceMaint) {
maintenance = true
continue
}
switch check.Status {
case "passing":
passing = true
case "warning":
warning = true
case "unknown":
unknown = true
case "critical":
critical = true
default:
return "", fmt.Errorf("unknown status: %q", check.Status)
}
}
switch {
case maintenance:
return HealthMaint, nil
case critical:
return HealthCritical, nil
case unknown:
return HealthUnknown, nil
case warning:
return HealthWarning, nil
case passing:
return HealthPassing, nil
default:
// No checks?
return HealthPassing, nil
}
// Type returns the type of this dependency.
func (d *HealthServiceQuery) Type() Type {
return TypeConsul
}
// ServiceStatusFilter is used to specify a list of service statuses that you want filter by.
type ServiceStatusFilter []string
// String returns the string representation of this status filter
func (f ServiceStatusFilter) String() string {
return fmt.Sprintf("[%s]", strings.Join(f, ","))
}
// NewServiceStatusFilter creates a status filter from the given string in the
// format `[key[,key[,key...]]]`. Each status is split on the comma character
// and must match one of the valid status names.
//
// If the empty string is given, it is assumed only "passing" statuses are to
// be returned.
//
// If the user specifies "any" with other keys, an error will be returned.
func NewServiceStatusFilter(s string) (ServiceStatusFilter, error) {
// If no statuses were given, use the default status of "all passing".
if len(s) == 0 || len(strings.TrimSpace(s)) == 0 {
return nil, nil
}
var errs *multierror.Error
var hasAny bool
raw := strings.Split(s, ",")
trimmed := make(ServiceStatusFilter, 0, len(raw))
for _, r := range raw {
trim := strings.TrimSpace(r)
// Ignore the empty string.
if len(trim) == 0 {
continue
}
// Record the case where we have the "any" status - it will be used later.
if trim == HealthAny {
hasAny = true
}
// Validate that the service is actually a valid name.
switch trim {
case HealthAny, HealthUnknown, HealthPassing, HealthWarning, HealthCritical, HealthMaint:
trimmed = append(trimmed, trim)
default:
errs = multierror.Append(errs, fmt.Errorf("service filter: invalid filter %q", trim))
}
}
// If the user specified "any" with additional keys, that is invalid.
if hasAny && len(trimmed) != 1 {
errs = multierror.Append(errs, fmt.Errorf("service filter: cannot specify extra keys when using %q", "any"))
}
return trimmed, errs.ErrorOrNil()
}
// Accept allows us to check if a slice of health checks pass this filter.
func (f ServiceStatusFilter) Accept(s string) bool {
// If the any filter is activated, pass everything.
if f.any() {
return true
}
// Iterate over each status and see if the given status is any of those
// statuses.
for _, status := range f {
if status == s {
// acceptStatus allows us to check if a slice of health checks pass this filter.
func acceptStatus(list []string, s string) bool {
for _, status := range list {
if status == s || status == HealthAny {
return true
}
}
return false
}
// any is a helper method to determine if this is an "any" service status
// filter. If "any" was given, it must be the only item in the list.
func (f ServiceStatusFilter) any() bool {
return len(f) == 1 && f[0] == HealthAny
}
// HealthServiceList is a sortable slice of Service
type HealthServiceList []*HealthService
// ByNodeThenID is a sortable slice of Service
type ByNodeThenID []*HealthService
// Len, Swap, and Less are used to implement the sort.Sort interface.
func (s HealthServiceList) Len() int { return len(s) }
func (s HealthServiceList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s HealthServiceList) Less(i, j int) bool {
func (s ByNodeThenID) Len() int { return len(s) }
func (s ByNodeThenID) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ByNodeThenID) Less(i, j int) bool {
if s[i].Node < s[j].Node {
return true
} else if s[i].Node == s[j].Node {

View File

@@ -0,0 +1,112 @@
package dependency
import (
"fmt"
"log"
"net/url"
"regexp"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*KVGetQuery)(nil)
// KVGetQueryRe is the regular expression to use.
KVGetQueryRe = regexp.MustCompile(`\A` + keyRe + dcRe + `\z`)
)
// KVGetQuery queries the KV store for a single key.
type KVGetQuery struct {
stopCh chan struct{}
dc string
key string
block bool
}
// NewKVGetQuery parses a string into a dependency.
func NewKVGetQuery(s string) (*KVGetQuery, error) {
if s != "" && !KVGetQueryRe.MatchString(s) {
return nil, fmt.Errorf("kv.get: invalid format: %q", s)
}
m := regexpMatch(KVGetQueryRe, s)
return &KVGetQuery{
dc: m["dc"],
key: m["key"],
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Consul API defined by the given client.
func (d *KVGetQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
default:
}
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
})
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/kv/" + d.key,
RawQuery: opts.String(),
})
pair, qm, err := clients.Consul().KV().Get(d.key, opts.ToConsulOpts())
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
LastContact: qm.LastContact,
Block: d.block,
}
if pair == nil {
log.Printf("[TRACE] %s: returned nil", d)
return nil, rm, nil
}
value := string(pair.Value)
log.Printf("[TRACE] %s: returned %q", d, value)
return value, rm, nil
}
// EnableBlocking turns this into a blocking KV query.
func (d *KVGetQuery) EnableBlocking() {
d.block = true
}
// CanShare returns a boolean if this dependency is shareable.
func (d *KVGetQuery) CanShare() bool {
return true
}
// String returns the human-friendly version of this dependency.
func (d *KVGetQuery) String() string {
key := d.key
if d.dc != "" {
key = key + "@" + d.dc
}
if d.block {
return fmt.Sprintf("kv.block(%s)", key)
}
return fmt.Sprintf("kv.get(%s)", key)
}
// Stop halts the dependency's fetch function.
func (d *KVGetQuery) Stop() {
close(d.stopCh)
}
// Type returns the type of this dependency.
func (d *KVGetQuery) Type() Type {
return TypeConsul
}

View File

@@ -0,0 +1,104 @@
package dependency
import (
"fmt"
"log"
"net/url"
"regexp"
"strings"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*KVKeysQuery)(nil)
// KVKeysQueryRe is the regular expression to use.
KVKeysQueryRe = regexp.MustCompile(`\A` + prefixRe + dcRe + `\z`)
)
// KVKeysQuery queries the KV store for a single key.
type KVKeysQuery struct {
stopCh chan struct{}
dc string
prefix string
}
// NewKVKeysQuery parses a string into a dependency.
func NewKVKeysQuery(s string) (*KVKeysQuery, error) {
if s != "" && !KVKeysQueryRe.MatchString(s) {
return nil, fmt.Errorf("kv.keys: invalid format: %q", s)
}
m := regexpMatch(KVKeysQueryRe, s)
return &KVKeysQuery{
dc: m["dc"],
prefix: m["prefix"],
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Consul API defined by the given client.
func (d *KVKeysQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
default:
}
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
})
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/kv/" + d.prefix,
RawQuery: opts.String(),
})
list, qm, err := clients.Consul().KV().Keys(d.prefix, "", opts.ToConsulOpts())
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
keys := make([]string, len(list))
for i, v := range list {
v = strings.TrimPrefix(v, d.prefix)
v = strings.TrimLeft(v, "/")
keys[i] = v
}
log.Printf("[TRACE] %s: returned %d results", d, len(list))
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
LastContact: qm.LastContact,
}
return keys, rm, nil
}
// CanShare returns a boolean if this dependency is shareable.
func (d *KVKeysQuery) CanShare() bool {
return true
}
// String returns the human-friendly version of this dependency.
func (d *KVKeysQuery) String() string {
prefix := d.prefix
if d.dc != "" {
prefix = prefix + "@" + d.dc
}
return fmt.Sprintf("kv.keys(%s)", prefix)
}
// Stop halts the dependency's fetch function.
func (d *KVKeysQuery) Stop() {
close(d.stopCh)
}
// Type returns the type of this dependency.
func (d *KVKeysQuery) Type() Type {
return TypeConsul
}

View File

@@ -0,0 +1,128 @@
package dependency
import (
"fmt"
"log"
"net/url"
"regexp"
"strings"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*KVListQuery)(nil)
// KVListQueryRe is the regular expression to use.
KVListQueryRe = regexp.MustCompile(`\A` + prefixRe + dcRe + `\z`)
)
// KeyPair is a simple Key-Value pair
type KeyPair struct {
Path string
Key string
Value string
// Lesser-used, but still valuable keys from api.KV
CreateIndex uint64
ModifyIndex uint64
LockIndex uint64
Flags uint64
Session string
}
// KVListQuery queries the KV store for a single key.
type KVListQuery struct {
stopCh chan struct{}
dc string
prefix string
}
// NewKVListQuery parses a string into a dependency.
func NewKVListQuery(s string) (*KVListQuery, error) {
if s != "" && !KVListQueryRe.MatchString(s) {
return nil, fmt.Errorf("kv.list: invalid format: %q", s)
}
m := regexpMatch(KVListQueryRe, s)
return &KVListQuery{
dc: m["dc"],
prefix: m["prefix"],
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Consul API defined by the given client.
func (d *KVListQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
default:
}
opts = opts.Merge(&QueryOptions{
Datacenter: d.dc,
})
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/kv/" + d.prefix,
RawQuery: opts.String(),
})
list, qm, err := clients.Consul().KV().List(d.prefix, opts.ToConsulOpts())
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
log.Printf("[TRACE] %s: returned %d pairs", d, len(list))
pairs := make([]*KeyPair, 0, len(list))
for _, pair := range list {
key := strings.TrimPrefix(pair.Key, d.prefix)
key = strings.TrimLeft(key, "/")
pairs = append(pairs, &KeyPair{
Path: pair.Key,
Key: key,
Value: string(pair.Value),
CreateIndex: pair.CreateIndex,
ModifyIndex: pair.ModifyIndex,
LockIndex: pair.LockIndex,
Flags: pair.Flags,
Session: pair.Session,
})
}
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
LastContact: qm.LastContact,
}
return pairs, rm, nil
}
// CanShare returns a boolean if this dependency is shareable.
func (d *KVListQuery) CanShare() bool {
return true
}
// String returns the human-friendly version of this dependency.
func (d *KVListQuery) String() string {
prefix := d.prefix
if d.dc != "" {
prefix = prefix + "@" + d.dc
}
return fmt.Sprintf("kv.list(%s)", prefix)
}
// Stop halts the dependency's fetch function.
func (d *KVListQuery) Stop() {
close(d.stopCh)
}
// Type returns the type of this dependency.
func (d *KVListQuery) Type() Type {
return TypeConsul
}

View File

@@ -0,0 +1,72 @@
package dependency
import (
"strings"
"sync"
)
// Set is a dependency-specific set implementation. Relative ordering is
// preserved.
type Set struct {
once sync.Once
sync.RWMutex
list []string
set map[string]Dependency
}
// Add adds a new element to the set if it does not already exist.
func (s *Set) Add(d Dependency) bool {
s.init()
s.Lock()
defer s.Unlock()
if _, ok := s.set[d.String()]; !ok {
s.list = append(s.list, d.String())
s.set[d.String()] = d
return true
}
return false
}
// Get retrieves a single element from the set by name.
func (s *Set) Get(v string) Dependency {
s.RLock()
defer s.RUnlock()
return s.set[v]
}
// List returns the insertion-ordered list of dependencies.
func (s *Set) List() []Dependency {
s.RLock()
defer s.RUnlock()
r := make([]Dependency, len(s.list))
for i, k := range s.list {
r[i] = s.set[k]
}
return r
}
// Len is the size of the set.
func (s *Set) Len() int {
s.RLock()
defer s.RUnlock()
return len(s.list)
}
// String is a string representation of the set.
func (s *Set) String() string {
s.RLock()
defer s.RUnlock()
return strings.Join(s.list, ", ")
}
func (s *Set) init() {
s.once.Do(func() {
if s.list == nil {
s.list = make([]string, 0, 8)
}
if s.set == nil {
s.set = make(map[string]Dependency)
}
})
}

View File

@@ -1,199 +0,0 @@
package dependency
import (
"errors"
"fmt"
"log"
"regexp"
"sync"
api "github.com/hashicorp/consul/api"
)
// StoreKey represents a single item in Consul's KV store.
type StoreKey struct {
sync.Mutex
rawKey string
Path string
DataCenter string
defaultValue string
defaultGiven bool
existenceCheck bool
stopped bool
stopCh chan struct{}
}
// kvGetResponse is a wrapper around the Consul API response.
type kvGetResponse struct {
pair *api.KVPair
meta *api.QueryMeta
err error
}
// Fetch queries the Consul API defined by the given client and returns string
// of the value to Path.
func (d *StoreKey) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
consulOpts := opts.consulQueryOptions()
if d.DataCenter != "" {
consulOpts.Datacenter = d.DataCenter
}
consul, err := clients.Consul()
if err != nil {
return nil, nil, fmt.Errorf("store key: error getting client: %s", err)
}
dataCh := make(chan *kvGetResponse, 1)
go func() {
log.Printf("[DEBUG] (%s) querying consul with %+v", d.Display(), consulOpts)
pair, meta, err := consul.KV().Get(d.Path, consulOpts)
resp := &kvGetResponse{pair: pair, meta: meta, err: err}
select {
case dataCh <- resp:
case <-d.stopCh:
}
}()
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case resp := <-dataCh:
if resp.err != nil {
return "", nil, fmt.Errorf("store key: error fetching: %s", resp.err)
}
rm := &ResponseMetadata{
LastIndex: resp.meta.LastIndex,
LastContact: resp.meta.LastContact,
}
if d.existenceCheck {
return (resp.pair != nil), rm, nil
}
if resp.pair == nil {
if d.defaultGiven {
log.Printf("[DEBUG] (%s) Consul returned no data (using default of %q)",
d.Display(), d.defaultValue)
return d.defaultValue, rm, nil
}
return nil, rm, nil
}
log.Printf("[DEBUG] (%s) Consul returned %s", d.Display(), resp.pair.Value)
return string(resp.pair.Value), rm, nil
}
}
// SetExistenceCheck sets this keys as an existence check instead of a value
// check.
func (d *StoreKey) SetExistenceCheck(b bool) {
d.existenceCheck = true
}
// SetDefault is used to set the default value.
func (d *StoreKey) SetDefault(s string) {
d.defaultGiven = true
d.defaultValue = s
}
// CanShare returns a boolean if this dependency is shareable.
func (d *StoreKey) CanShare() bool {
return true
}
// HashCode returns a unique identifier.
func (d *StoreKey) HashCode() string {
if d.existenceCheck {
return fmt.Sprintf("StoreKeyExists|%s", d.rawKey)
}
if d.defaultGiven {
return fmt.Sprintf("StoreKey|%s|%s", d.rawKey, d.defaultValue)
}
return fmt.Sprintf("StoreKey|%s", d.rawKey)
}
// Display prints the human-friendly output.
func (d *StoreKey) Display() string {
if d.existenceCheck {
return fmt.Sprintf(`"key_exists(%s)"`, d.rawKey)
}
if d.defaultGiven {
return fmt.Sprintf(`"key_or_default(%s, %q)"`, d.rawKey, d.defaultValue)
}
return fmt.Sprintf(`"key(%s)"`, d.rawKey)
}
// Stop halts the dependency's fetch function.
func (d *StoreKey) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
}
// ParseStoreKey parses a string of the format a(/b(/c...))
func ParseStoreKey(s string) (*StoreKey, error) {
if len(s) == 0 {
return nil, errors.New("cannot specify empty key dependency")
}
re := regexp.MustCompile(`\A` +
`(?P<key>[^@]+)` +
`(@(?P<datacenter>.+))?` +
`\z`)
names := re.SubexpNames()
match := re.FindAllStringSubmatch(s, -1)
if len(match) == 0 {
return nil, errors.New("invalid key dependency format")
}
r := match[0]
m := map[string]string{}
for i, n := range r {
if names[i] != "" {
m[names[i]] = n
}
}
key, datacenter := m["key"], m["datacenter"]
if key == "" {
return nil, errors.New("key part is required")
}
kd := &StoreKey{
rawKey: s,
Path: key,
DataCenter: datacenter,
stopCh: make(chan struct{}),
}
return kd, nil
}

View File

@@ -1,184 +0,0 @@
package dependency
import (
"encoding/gob"
"errors"
"fmt"
"log"
"regexp"
"strings"
"sync"
"github.com/hashicorp/consul/api"
)
func init() {
gob.Register([]*KeyPair{})
}
// KeyPair is a simple Key-Value pair
type KeyPair struct {
Path string
Key string
Value string
// Lesser-used, but still valuable keys from api.KV
CreateIndex uint64
ModifyIndex uint64
LockIndex uint64
Flags uint64
Session string
}
// StoreKeyPrefix is the representation of a requested key dependency
// from inside a template.
type StoreKeyPrefix struct {
sync.Mutex
rawKey string
Prefix string
DataCenter string
stopped bool
stopCh chan struct{}
}
// Fetch queries the Consul API defined by the given client and returns a slice
// of KeyPair objects
func (d *StoreKeyPrefix) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
consulOpts := opts.consulQueryOptions()
if d.DataCenter != "" {
consulOpts.Datacenter = d.DataCenter
}
consul, err := clients.Consul()
if err != nil {
return nil, nil, fmt.Errorf("store key prefix: error getting client: %s", err)
}
var prefixes api.KVPairs
var qm *api.QueryMeta
dataCh := make(chan struct{})
go func() {
log.Printf("[DEBUG] (%s) querying consul with %+v", d.Display(), consulOpts)
prefixes, qm, err = consul.KV().List(d.Prefix, consulOpts)
close(dataCh)
}()
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-dataCh:
}
if err != nil {
return nil, nil, fmt.Errorf("store key prefix: error fetching: %s", err)
}
log.Printf("[DEBUG] (%s) Consul returned %d key pairs", d.Display(), len(prefixes))
keyPairs := make([]*KeyPair, 0, len(prefixes))
for _, pair := range prefixes {
key := strings.TrimPrefix(pair.Key, d.Prefix)
key = strings.TrimLeft(key, "/")
keyPairs = append(keyPairs, &KeyPair{
Path: pair.Key,
Key: key,
Value: string(pair.Value),
CreateIndex: pair.CreateIndex,
ModifyIndex: pair.ModifyIndex,
LockIndex: pair.LockIndex,
Flags: pair.Flags,
Session: pair.Session,
})
}
rm := &ResponseMetadata{
LastIndex: qm.LastIndex,
LastContact: qm.LastContact,
}
return keyPairs, rm, nil
}
// CanShare returns a boolean if this dependency is shareable.
func (d *StoreKeyPrefix) CanShare() bool {
return true
}
// HashCode returns a unique identifier.
func (d *StoreKeyPrefix) HashCode() string {
return fmt.Sprintf("StoreKeyPrefix|%s", d.rawKey)
}
// Display prints the human-friendly output.
func (d *StoreKeyPrefix) Display() string {
return fmt.Sprintf(`"storeKeyPrefix(%s)"`, d.rawKey)
}
// Stop halts the dependency's fetch function.
func (d *StoreKeyPrefix) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
}
// ParseStoreKeyPrefix parses a string of the format a(/b(/c...))
func ParseStoreKeyPrefix(s string) (*StoreKeyPrefix, error) {
// a(/b(/c))(@datacenter)
re := regexp.MustCompile(`\A` +
`(?P<prefix>[[:word:],\.\:\-\/]+)?` +
`(@(?P<datacenter>[[:word:]\.\-]+))?` +
`\z`)
names := re.SubexpNames()
match := re.FindAllStringSubmatch(s, -1)
if len(match) == 0 {
return nil, errors.New("invalid key prefix dependency format")
}
r := match[0]
m := map[string]string{}
for i, n := range r {
if names[i] != "" {
m[names[i]] = n
}
}
prefix, datacenter := m["prefix"], m["datacenter"]
// Empty prefix or nil prefix should default to "/"
if len(prefix) == 0 {
prefix = "/"
}
// Remove leading slash
if len(prefix) > 1 && prefix[0] == '/' {
prefix = prefix[1:len(prefix)]
}
kpd := &StoreKeyPrefix{
rawKey: s,
Prefix: prefix,
DataCenter: datacenter,
stopCh: make(chan struct{}),
}
return kpd, nil
}

View File

@@ -1,126 +0,0 @@
package dependency
import (
"fmt"
"sync"
"time"
)
// Test is a special dependency that does not actually speaks to a server.
type Test struct {
Name string
}
func (d *Test) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
time.Sleep(10 * time.Millisecond)
data := "this is some data"
rm := &ResponseMetadata{LastIndex: 1}
return data, rm, nil
}
func (d *Test) CanShare() bool {
return true
}
func (d *Test) HashCode() string {
return fmt.Sprintf("Test|%s", d.Name)
}
func (d *Test) Display() string { return "fakedep" }
func (d *Test) Stop() {}
// TestStale is a special dependency that can be used to test what happens when
// stale data is permitted.
type TestStale struct {
Name string
}
// Fetch is used to implement the dependency interface.
func (d *TestStale) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
time.Sleep(10 * time.Millisecond)
if opts == nil {
opts = &QueryOptions{}
}
if opts.AllowStale {
data := "this is some stale data"
rm := &ResponseMetadata{LastIndex: 1, LastContact: 50 * time.Millisecond}
return data, rm, nil
} else {
data := "this is some fresh data"
rm := &ResponseMetadata{LastIndex: 1}
return data, rm, nil
}
}
func (d *TestStale) CanShare() bool {
return true
}
func (d *TestStale) HashCode() string {
return fmt.Sprintf("TestStale|%s", d.Name)
}
func (d *TestStale) Display() string { return "fakedep" }
func (d *TestStale) Stop() {}
// TestFetchError is a special dependency that returns an error while fetching.
type TestFetchError struct {
Name string
}
func (d *TestFetchError) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
time.Sleep(10 * time.Millisecond)
return nil, nil, fmt.Errorf("failed to contact server")
}
func (d *TestFetchError) CanShare() bool {
return true
}
func (d *TestFetchError) HashCode() string {
return fmt.Sprintf("TestFetchError|%s", d.Name)
}
func (d *TestFetchError) Display() string { return "fakedep" }
func (d *TestFetchError) Stop() {}
// TestRetry is a special dependency that errors on the first fetch and
// succeeds on subsequent fetches.
type TestRetry struct {
sync.Mutex
Name string
retried bool
}
func (d *TestRetry) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
time.Sleep(10 * time.Millisecond)
d.Lock()
defer d.Unlock()
if d.retried {
data := "this is some data"
rm := &ResponseMetadata{LastIndex: 1}
return data, rm, nil
} else {
d.retried = true
return nil, nil, fmt.Errorf("failed to contact server (try again)")
}
}
func (d *TestRetry) CanShare() bool {
return true
}
func (d *TestRetry) HashCode() string {
return fmt.Sprintf("TestRetry|%s", d.Name)
}
func (d *TestRetry) Display() string { return "fakedep" }
func (d *TestRetry) Stop() {}

View File

@@ -0,0 +1,26 @@
package dependency
var (
// VaultDefaultLeaseDuration is the default lease duration in seconds.
VaultDefaultLeaseDuration = 5 * 60
)
// Secret is a vault secret.
type Secret struct {
RequestID string
LeaseID string
LeaseDuration int
Renewable bool
// Data is the actual contents of the secret. The format of the data
// is arbitrary and up to the secret backend.
Data map[string]interface{}
}
// leaseDurationOrDefault returns a value or the default lease duration.
func leaseDurationOrDefault(d int) int {
if d == 0 {
return VaultDefaultLeaseDuration
}
return d
}

View File

@@ -0,0 +1,135 @@
package dependency
import (
"fmt"
"log"
"net/url"
"sort"
"strings"
"time"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*VaultListQuery)(nil)
)
// VaultListQuery is the dependency to Vault for a secret
type VaultListQuery struct {
stopCh chan struct{}
path string
secret *Secret
}
// NewVaultListQuery creates a new datacenter dependency.
func NewVaultListQuery(s string) (*VaultListQuery, error) {
s = strings.TrimSpace(s)
s = strings.Trim(s, "/")
if s == "" {
return nil, fmt.Errorf("vault.list: invalid format: %q", s)
}
return &VaultListQuery{
path: s,
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Vault API
func (d *VaultListQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
default:
}
opts = opts.Merge(&QueryOptions{})
// If this is not the first query, poll to simulate blocking-queries.
if opts.WaitIndex != 0 {
dur := time.Duration(d.secret.LeaseDuration/2.0) * time.Second
if dur == 0 {
dur = time.Duration(VaultDefaultLeaseDuration)
}
log.Printf("[TRACE] %s: long polling for %s", d, dur)
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-time.After(dur):
}
}
// If we got this far, we either didn't have a secret to renew, the secret was
// not renewable, or the renewal failed, so attempt a fresh list.
log.Printf("[TRACE] %s: LIST %s", d, &url.URL{
Path: "/v1/" + d.path,
RawQuery: opts.String(),
})
secret, err := clients.Vault().Logical().List(d.path)
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
// The secret could be nil if it does not exist.
if secret == nil || secret.Data == nil {
return respWithMetadata([]string{})
}
// This is a weird thing that happened once...
keys, ok := secret.Data["keys"]
if !ok {
return respWithMetadata([]string{})
}
list, ok := keys.([]interface{})
if !ok {
return nil, nil, fmt.Errorf("%s: unexpected response", d)
}
result := make([]string, len(list))
for i, v := range list {
typed, ok := v.(string)
if !ok {
return nil, nil, fmt.Errorf("%s: non-string in list", d)
}
result[i] = typed
}
sort.Strings(result)
d.secret = &Secret{
RequestID: secret.RequestID,
LeaseID: secret.LeaseID,
LeaseDuration: secret.LeaseDuration,
Renewable: secret.Renewable,
Data: secret.Data,
}
log.Printf("[TRACE] %s: returned %d results", d, len(result))
return respWithMetadata(result)
}
// CanShare returns if this dependency is shareable.
func (d *VaultListQuery) CanShare() bool {
return false
}
// Stop halts the given dependency's fetch.
func (d *VaultListQuery) Stop() {
close(d.stopCh)
}
// String returns the human-friendly version of this dependency.
func (d *VaultListQuery) String() string {
return fmt.Sprintf("vault.list(%s)", d.path)
}
// Type returns the type of this dependency.
func (d *VaultListQuery) Type() Type {
return TypeVault
}

View File

@@ -0,0 +1,147 @@
package dependency
import (
"fmt"
"log"
"net/url"
"strings"
"time"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*VaultReadQuery)(nil)
)
// VaultReadQuery is the dependency to Vault for a secret
type VaultReadQuery struct {
stopCh chan struct{}
path string
secret *Secret
}
// NewVaultReadQuery creates a new datacenter dependency.
func NewVaultReadQuery(s string) (*VaultReadQuery, error) {
s = strings.TrimSpace(s)
s = strings.Trim(s, "/")
if s == "" {
return nil, fmt.Errorf("vault.read: invalid format: %q", s)
}
return &VaultReadQuery{
path: s,
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Vault API
func (d *VaultReadQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
default:
}
opts = opts.Merge(&QueryOptions{})
// If this is not the first query and we have a lease duration, sleep until we
// try to renew.
if opts.WaitIndex != 0 && d.secret != nil && d.secret.LeaseDuration != 0 {
dur := time.Duration(d.secret.LeaseDuration/2.0) * time.Second
if dur == 0 {
dur = time.Duration(VaultDefaultLeaseDuration)
}
log.Printf("[TRACE] %s: long polling for %s", d, dur)
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-time.After(dur):
}
}
// Attempt to renew the secret. If we do not have a secret or if that secret
// is not renewable, we will attempt a (re-)read later.
if d.secret != nil && d.secret.LeaseID != "" && d.secret.Renewable {
log.Printf("[TRACE] %s: PUT %s", d, &url.URL{
Path: "/v1/sys/renew/" + d.secret.LeaseID,
RawQuery: opts.String(),
})
renewal, err := clients.Vault().Sys().Renew(d.secret.LeaseID, 0)
if err == nil {
log.Printf("[TRACE] %s: successfully renewed %s", d, d.secret.LeaseID)
secret := &Secret{
RequestID: renewal.RequestID,
LeaseID: renewal.LeaseID,
LeaseDuration: d.secret.LeaseDuration,
Renewable: renewal.Renewable,
Data: d.secret.Data,
}
d.secret = secret
return respWithMetadata(secret)
}
// The renewal failed for some reason.
log.Printf("[WARN] %s: failed to renew %s: %s", d, d.secret.LeaseID, err)
}
// If we got this far, we either didn't have a secret to renew, the secret was
// not renewable, or the renewal failed, so attempt a fresh read.
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/" + d.path,
RawQuery: opts.String(),
})
vaultSecret, err := clients.Vault().Logical().Read(d.path)
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
// The secret could be nil if it does not exist.
if vaultSecret == nil {
log.Printf("[WARN] %s: returned nil (does the secret exist?)", d)
return respWithMetadata(nil)
}
// Print any warnings.
for _, w := range vaultSecret.Warnings {
log.Printf("[WARN] %s: %s", d, w)
}
// Create our cloned secret.
secret := &Secret{
LeaseID: vaultSecret.LeaseID,
LeaseDuration: leaseDurationOrDefault(vaultSecret.LeaseDuration),
Renewable: vaultSecret.Renewable,
Data: vaultSecret.Data,
}
d.secret = secret
return respWithMetadata(secret)
}
// CanShare returns if this dependency is shareable.
func (d *VaultReadQuery) CanShare() bool {
return false
}
// Stop halts the given dependency's fetch.
func (d *VaultReadQuery) Stop() {
close(d.stopCh)
}
// String returns the human-friendly version of this dependency.
func (d *VaultReadQuery) String() string {
return fmt.Sprintf("vault.read(%s)", d.path)
}
// Type returns the type of this dependency.
func (d *VaultReadQuery) Type() Type {
return TypeVault
}

View File

@@ -1,197 +0,0 @@
package dependency
import (
"fmt"
"log"
"strings"
"sync"
"time"
vaultapi "github.com/hashicorp/vault/api"
)
// Secret is a vault secret.
type Secret struct {
LeaseID string
LeaseDuration int
Renewable bool
// Data is the actual contents of the secret. The format of the data
// is arbitrary and up to the secret backend.
Data map[string]interface{}
}
// VaultSecret is the dependency to Vault for a secret
type VaultSecret struct {
sync.Mutex
Path string
data map[string]interface{}
secret *Secret
stopped bool
stopCh chan struct{}
}
// Fetch queries the Vault API
func (d *VaultSecret) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
log.Printf("[DEBUG] (%s) querying vault with %+v", d.Display(), opts)
// If this is not the first query and we have a lease duration, sleep until we
// try to renew.
if opts.WaitIndex != 0 && d.secret != nil && d.secret.LeaseDuration != 0 {
duration := time.Duration(d.secret.LeaseDuration/2.0) * time.Second
log.Printf("[DEBUG] (%s) pretending to long-poll for %q",
d.Display(), duration)
select {
case <-d.stopCh:
log.Printf("[DEBUG] (%s) received interrupt", d.Display())
return nil, nil, ErrStopped
case <-time.After(duration):
}
}
// Grab the vault client
vault, err := clients.Vault()
if err != nil {
return nil, nil, ErrWithExitf("vault secret: %s", err)
}
// Attempt to renew the secret. If we do not have a secret or if that secret
// is not renewable, we will attempt a (re-)read later.
if d.secret != nil && d.secret.LeaseID != "" && d.secret.Renewable {
renewal, err := vault.Sys().Renew(d.secret.LeaseID, 0)
if err == nil {
log.Printf("[DEBUG] (%s) successfully renewed", d.Display())
log.Printf("[DEBUG] (%s) %#v", d.Display(), renewal)
secret := &Secret{
LeaseID: renewal.LeaseID,
LeaseDuration: d.secret.LeaseDuration,
Renewable: renewal.Renewable,
Data: d.secret.Data,
}
d.Lock()
d.secret = secret
d.Unlock()
return respWithMetadata(secret)
}
// The renewal failed for some reason.
log.Printf("[WARN] (%s) failed to renew, re-obtaining: %s", d.Display(), err)
}
// If we got this far, we either didn't have a secret to renew, the secret was
// not renewable, or the renewal failed, so attempt a fresh read.
var vaultSecret *vaultapi.Secret
if len(d.data) == 0 {
vaultSecret, err = vault.Logical().Read(d.Path)
} else {
vaultSecret, err = vault.Logical().Write(d.Path, d.data)
}
if err != nil {
return nil, nil, ErrWithExitf("error obtaining from vault: %s", err)
}
// The secret could be nil (maybe it does not exist yet). This is not an error
// to Vault, but it is an error to Consul Template, so return an error
// instead.
if vaultSecret == nil {
return nil, nil, fmt.Errorf("no secret exists at path %q", d.Display())
}
// Create our cloned secret
secret := &Secret{
LeaseID: vaultSecret.LeaseID,
LeaseDuration: leaseDurationOrDefault(vaultSecret.LeaseDuration),
Renewable: vaultSecret.Renewable,
Data: vaultSecret.Data,
}
d.Lock()
d.secret = secret
d.Unlock()
log.Printf("[DEBUG] (%s) vault returned the secret", d.Display())
return respWithMetadata(secret)
}
// CanShare returns if this dependency is shareable.
func (d *VaultSecret) CanShare() bool {
return false
}
// HashCode returns the hash code for this dependency.
func (d *VaultSecret) HashCode() string {
return fmt.Sprintf("VaultSecret|%s", d.Path)
}
// Display returns a string that should be displayed to the user in output (for
// example).
func (d *VaultSecret) Display() string {
return fmt.Sprintf(`"secret(%s)"`, d.Path)
}
// Stop halts the given dependency's fetch.
func (d *VaultSecret) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
}
// ParseVaultSecret creates a new datacenter dependency.
func ParseVaultSecret(s ...string) (*VaultSecret, error) {
if len(s) == 0 {
return nil, fmt.Errorf("expected 1 or more arguments, got %d", len(s))
}
path, rest := s[0], s[1:len(s)]
if len(path) == 0 {
return nil, fmt.Errorf("vault path must be at least one character")
}
data := make(map[string]interface{})
for _, str := range rest {
parts := strings.SplitN(str, "=", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid value %q - must be key=value", str)
}
k, v := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])
data[k] = v
}
vs := &VaultSecret{
Path: path,
data: data,
stopCh: make(chan struct{}),
}
return vs, nil
}
func leaseDurationOrDefault(d int) int {
if d == 0 {
return 5 * 60
}
return d
}

View File

@@ -1,134 +0,0 @@
package dependency
import (
"fmt"
"log"
"sort"
"sync"
"time"
)
// VaultSecrets is the dependency to list secrets in Vault.
type VaultSecrets struct {
sync.Mutex
Path string
stopped bool
stopCh chan struct{}
}
// Fetch queries the Vault API
func (d *VaultSecrets) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
}
log.Printf("[DEBUG] (%s) querying vault with %+v", d.Display(), opts)
// If this is not the first query and we have a lease duration, sleep until we
// try to renew.
if opts.WaitIndex != 0 {
log.Printf("[DEBUG] (%s) pretending to long-poll", d.Display())
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-time.After(sleepTime):
}
}
// Grab the vault client
vault, err := clients.Vault()
if err != nil {
return nil, nil, ErrWithExitf("vault secrets: %s", err)
}
// Get the list as a secret
vaultSecret, err := vault.Logical().List(d.Path)
if err != nil {
return nil, nil, ErrWithExitf("error listing secrets from vault: %s", err)
}
// If the secret or data data is nil, return an empty list of strings.
if vaultSecret == nil || vaultSecret.Data == nil {
return respWithMetadata(make([]string, 0))
}
// If there are no keys at that path, return the empty list.
keys, ok := vaultSecret.Data["keys"]
if !ok {
return respWithMetadata(make([]string, 0))
}
// Convert the interface into a list of interfaces.
list, ok := keys.([]interface{})
if !ok {
return nil, nil, ErrWithExitf("vault returned an unexpected payload for %q", d.Display())
}
// Pull each item out of the list and safely cast to a string.
result := make([]string, len(list))
for i, v := range list {
typed, ok := v.(string)
if !ok {
return nil, nil, ErrWithExitf("vault returned a non-string when listing secrets for %q", d.Display())
}
result[i] = typed
}
sort.Strings(result)
log.Printf("[DEBUG] (%s) vault listed %d secrets(s)", d.Display(), len(result))
return respWithMetadata(result)
}
// CanShare returns if this dependency is shareable.
func (d *VaultSecrets) CanShare() bool {
return false
}
// HashCode returns the hash code for this dependency.
func (d *VaultSecrets) HashCode() string {
return fmt.Sprintf("VaultSecrets|%s", d.Path)
}
// Display returns a string that should be displayed to the user in output (for
// example).
func (d *VaultSecrets) Display() string {
return fmt.Sprintf(`"secrets(%s)"`, d.Path)
}
// Stop halts the dependency's fetch function.
func (d *VaultSecrets) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
}
// ParseVaultSecrets creates a new datacenter dependency.
func ParseVaultSecrets(s string) (*VaultSecrets, error) {
// Ensure a trailing slash, always.
if len(s) == 0 {
s = "/"
}
if s[len(s)-1] != '/' {
s = fmt.Sprintf("%s/", s)
}
vs := &VaultSecrets{
Path: s,
stopCh: make(chan struct{}),
}
return vs, nil
}

View File

@@ -2,64 +2,67 @@ package dependency
import (
"log"
"sync"
"net/url"
"time"
"github.com/pkg/errors"
)
// VaultToken is the dependency to Vault for a secret
type VaultToken struct {
sync.Mutex
var (
// Ensure implements
_ Dependency = (*VaultTokenQuery)(nil)
)
// VaultTokenQuery is the dependency to Vault for a secret
type VaultTokenQuery struct {
stopCh chan struct{}
leaseID string
leaseDuration int
}
stopped bool
stopCh chan struct{}
// NewVaultTokenQuery creates a new dependency.
func NewVaultTokenQuery() (*VaultTokenQuery, error) {
return &VaultTokenQuery{
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Vault API
func (d *VaultToken) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
d.Lock()
if d.stopped {
defer d.Unlock()
func (d *VaultTokenQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
}
d.Unlock()
if opts == nil {
opts = &QueryOptions{}
default:
}
log.Printf("[DEBUG] (%s) renewing vault token", d.Display())
opts = opts.Merge(&QueryOptions{})
log.Printf("[TRACE] %s: GET %s", d, &url.URL{
Path: "/v1/auth/token/renew-self",
RawQuery: opts.String(),
})
// If this is not the first query and we have a lease duration, sleep until we
// try to renew.
if opts.WaitIndex != 0 && d.leaseDuration != 0 {
duration := time.Duration(d.leaseDuration/2.0) * time.Second
if duration < 1*time.Second {
log.Printf("[DEBUG] (%s) increasing sleep to 1s (was %q)",
d.Display(), duration)
duration = 1 * time.Second
dur := time.Duration(d.leaseDuration/2.0) * time.Second
if dur == 0 {
dur = time.Duration(VaultDefaultLeaseDuration)
}
log.Printf("[DEBUG] (%s) sleeping for %q", d.Display(), duration)
log.Printf("[TRACE] %s: long polling for %s", d, dur)
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-time.After(duration):
case <-time.After(dur):
}
}
// Grab the vault client
vault, err := clients.Vault()
token, err := clients.Vault().Auth().Token().RenewSelf(0)
if err != nil {
return nil, nil, ErrWithExitf("vault_token: %s", err)
}
token, err := vault.Auth().Token().RenewSelf(0)
if err != nil {
return nil, nil, ErrWithExitf("error renewing vault token: %s", err)
return nil, nil, errors.Wrap(err, d.String())
}
// Create our cloned secret
@@ -70,50 +73,30 @@ func (d *VaultToken) Fetch(clients *ClientSet, opts *QueryOptions) (interface{},
Data: token.Data,
}
leaseDuration := token.Auth.LeaseDuration
if leaseDuration == 0 {
log.Printf("[WARN] (%s) lease duration is 0, setting to 5s", d.Display())
leaseDuration = 5
}
d.Lock()
d.leaseID = secret.LeaseID
d.leaseDuration = leaseDuration
d.Unlock()
d.leaseDuration = secret.LeaseDuration
log.Printf("[DEBUG] (%s) successfully renewed token", d.Display())
log.Printf("[DEBUG] %s: renewed token", d)
return respWithMetadata(secret)
}
// CanShare returns if this dependency is shareable.
func (d *VaultToken) CanShare() bool {
func (d *VaultTokenQuery) CanShare() bool {
return false
}
// HashCode returns the hash code for this dependency.
func (d *VaultToken) HashCode() string {
return "VaultToken"
}
// Display returns a string that should be displayed to the user in output (for
// example).
func (d *VaultToken) Display() string {
return "vault_token"
}
// Stop halts the dependency's fetch function.
func (d *VaultToken) Stop() {
d.Lock()
defer d.Unlock()
if !d.stopped {
close(d.stopCh)
d.stopped = true
}
func (d *VaultTokenQuery) Stop() {
close(d.stopCh)
}
// ParseVaultToken creates a new VaultToken dependency.
func ParseVaultToken() (*VaultToken, error) {
return &VaultToken{stopCh: make(chan struct{})}, nil
// String returns the human-friendly version of this dependency.
func (d *VaultTokenQuery) String() string {
return "vault.token"
}
// Type returns the type of this dependency.
func (d *VaultTokenQuery) Type() Type {
return TypeVault
}

View File

@@ -0,0 +1,173 @@
package dependency
import (
"crypto/sha1"
"fmt"
"io"
"log"
"net/url"
"sort"
"strings"
"time"
"github.com/pkg/errors"
)
var (
// Ensure implements
_ Dependency = (*VaultWriteQuery)(nil)
)
// VaultWriteQuery is the dependency to Vault for a secret
type VaultWriteQuery struct {
stopCh chan struct{}
path string
data map[string]interface{}
dataHash string
secret *Secret
}
// NewVaultWriteQuery creates a new datacenter dependency.
func NewVaultWriteQuery(s string, d map[string]interface{}) (*VaultWriteQuery, error) {
s = strings.TrimSpace(s)
s = strings.Trim(s, "/")
if s == "" {
return nil, fmt.Errorf("vault.write: invalid format: %q", s)
}
return &VaultWriteQuery{
path: s,
data: d,
dataHash: sha1Map(d),
stopCh: make(chan struct{}, 1),
}, nil
}
// Fetch queries the Vault API
func (d *VaultWriteQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
select {
case <-d.stopCh:
return nil, nil, ErrStopped
default:
}
opts = opts.Merge(&QueryOptions{})
// If this is not the first query and we have a lease duration, sleep until we
// try to renew.
if opts.WaitIndex != 0 && d.secret != nil && d.secret.LeaseDuration != 0 {
dur := time.Duration(d.secret.LeaseDuration/2.0) * time.Second
if dur == 0 {
dur = time.Duration(VaultDefaultLeaseDuration)
}
log.Printf("[TRACE] %s: long polling for %s", d, dur)
select {
case <-d.stopCh:
return nil, nil, ErrStopped
case <-time.After(dur):
}
}
// Attempt to renew the secret. If we do not have a secret or if that secret
// is not renewable, we will attempt a (re-)write later.
if d.secret != nil && d.secret.LeaseID != "" && d.secret.Renewable {
log.Printf("[TRACE] %s: PUT %s", d, &url.URL{
Path: "/v1/sys/renew/" + d.secret.LeaseID,
RawQuery: opts.String(),
})
renewal, err := clients.Vault().Sys().Renew(d.secret.LeaseID, 0)
if err == nil {
log.Printf("[TRACE] %s: successfully renewed %s", d, d.secret.LeaseID)
secret := &Secret{
RequestID: renewal.RequestID,
LeaseID: renewal.LeaseID,
LeaseDuration: d.secret.LeaseDuration,
Renewable: renewal.Renewable,
Data: d.secret.Data,
}
d.secret = secret
return respWithMetadata(secret)
}
// The renewal failed for some reason.
log.Printf("[WARN] %s: failed to renew %s: %s", d, d.secret.LeaseID, err)
}
// If we got this far, we either didn't have a secret to renew, the secret was
// not renewable, or the renewal failed, so attempt a fresh write.
log.Printf("[TRACE] %s: PUT %s", d, &url.URL{
Path: "/v1/" + d.path,
RawQuery: opts.String(),
})
vaultSecret, err := clients.Vault().Logical().Write(d.path, d.data)
if err != nil {
return nil, nil, errors.Wrap(err, d.String())
}
// The secret could be nil if it does not exist.
if vaultSecret == nil {
log.Printf("[WARN] %s: returned nil (does the secret exist?)", d)
return respWithMetadata(nil)
}
// Print any warnings.
for _, w := range vaultSecret.Warnings {
log.Printf("[WARN] %s: %s", d, w)
}
// Create our cloned secret.
secret := &Secret{
LeaseID: vaultSecret.LeaseID,
LeaseDuration: leaseDurationOrDefault(vaultSecret.LeaseDuration),
Renewable: vaultSecret.Renewable,
Data: vaultSecret.Data,
}
d.secret = secret
return respWithMetadata(secret)
}
// CanShare returns if this dependency is shareable.
func (d *VaultWriteQuery) CanShare() bool {
return false
}
// Stop halts the given dependency's fetch.
func (d *VaultWriteQuery) Stop() {
close(d.stopCh)
}
// String returns the human-friendly version of this dependency.
func (d *VaultWriteQuery) String() string {
return fmt.Sprintf("vault.write(%s -> %s)", d.path, d.dataHash)
}
// Type returns the type of this dependency.
func (d *VaultWriteQuery) Type() Type {
return TypeVault
}
// sha1Map returns the sha1 hash of the data in the map. The reason this data is
// hashed is because it appears in the output and could contain sensitive
// information.
func sha1Map(m map[string]interface{}) string {
keys := make([]string, 0, len(m))
for k, _ := range m {
keys = append(keys, k)
}
sort.Strings(keys)
h := sha1.New()
for _, k := range keys {
io.WriteString(h, fmt.Sprintf("%s=%q", k, m[k]))
}
return fmt.Sprintf("%.4x", h.Sum(nil))
}

View File

@@ -57,8 +57,8 @@ type templateData struct {
// path for a total of 100.
//
type DedupManager struct {
// config is the consul-template configuration
config *config.Config
// config is the deduplicate configuration
config *config.DedupConfig
// clients is used to access the underlying clinets
clients *dep.ClientSet
@@ -89,7 +89,7 @@ type DedupManager struct {
}
// NewDedupManager creates a new Dedup manager
func NewDedupManager(config *config.Config, clients *dep.ClientSet, brain *template.Brain, templates []*template.Template) (*DedupManager, error) {
func NewDedupManager(config *config.DedupConfig, clients *dep.ClientSet, brain *template.Brain, templates []*template.Template) (*DedupManager, error) {
d := &DedupManager{
config: config,
clients: clients,
@@ -107,10 +107,7 @@ func NewDedupManager(config *config.Config, clients *dep.ClientSet, brain *templ
func (d *DedupManager) Start() error {
log.Printf("[INFO] (dedup) starting de-duplication manager")
client, err := d.clients.Consul()
if err != nil {
return err
}
client := d.clients.Consul()
go d.createSession(client)
// Start to watch each template
@@ -141,7 +138,7 @@ START:
log.Printf("[INFO] (dedup) attempting to create session")
session := client.Session()
sessionCh := make(chan struct{})
ttl := fmt.Sprintf("%ds", d.config.Deduplicate.TTL/time.Second)
ttl := fmt.Sprintf("%.6fs", float64(*d.config.TTL)/float64(time.Second))
se := &consulapi.SessionEntry{
Name: "Consul-Template de-duplication",
Behavior: "delete",
@@ -196,7 +193,7 @@ func (d *DedupManager) IsLeader(tmpl *template.Template) bool {
// UpdateDeps is used to update the values of the dependencies for a template
func (d *DedupManager) UpdateDeps(t *template.Template, deps []dep.Dependency) error {
// Calculate the path to write updates to
dataPath := path.Join(d.config.Deduplicate.Prefix, t.HexMD5, "data")
dataPath := path.Join(*d.config.Prefix, t.ID(), "data")
// Package up the dependency data
td := templateData{
@@ -211,7 +208,7 @@ func (d *DedupManager) UpdateDeps(t *template.Template, deps []dep.Dependency) e
// Pull the current value from the brain
val, ok := d.brain.Recall(dp)
if ok {
td.Data[dp.HashCode()] = val
td.Data[dp.String()] = val
}
}
@@ -241,10 +238,7 @@ func (d *DedupManager) UpdateDeps(t *template.Template, deps []dep.Dependency) e
Value: buf.Bytes(),
Flags: templateDataFlag,
}
client, err := d.clients.Consul()
if err != nil {
return fmt.Errorf("failed to get consul client: %v", err)
}
client := d.clients.Consul()
if _, err := client.KV().Put(&kvPair, nil); err != nil {
return fmt.Errorf("failed to write '%s': %v", dataPath, err)
}
@@ -286,12 +280,12 @@ func (d *DedupManager) setLeader(tmpl *template.Template, lockCh <-chan struct{}
}
func (d *DedupManager) watchTemplate(client *consulapi.Client, t *template.Template) {
log.Printf("[INFO] (dedup) starting watch for template hash %s", t.HexMD5)
path := path.Join(d.config.Deduplicate.Prefix, t.HexMD5, "data")
log.Printf("[INFO] (dedup) starting watch for template hash %s", t.ID())
path := path.Join(*d.config.Prefix, t.ID(), "data")
// Determine if stale queries are allowed
var allowStale bool
if d.config.MaxStale != 0 {
if *d.config.MaxStale != 0 {
allowStale = true
}
@@ -323,7 +317,7 @@ START:
}
// Block for updates on the data key
log.Printf("[INFO] (dedup) listing data for template hash %s", t.HexMD5)
log.Printf("[INFO] (dedup) listing data for template hash %s", t.ID())
pair, meta, err := client.KV().Get(path, opts)
if err != nil {
log.Printf("[ERR] (dedup) failed to get '%s': %v", path, err)
@@ -337,14 +331,14 @@ START:
opts.WaitIndex = meta.LastIndex
// If we've exceeded the maximum staleness, retry without stale
if allowStale && meta.LastContact > d.config.MaxStale {
if allowStale && meta.LastContact > *d.config.MaxStale {
allowStale = false
log.Printf("[DEBUG] (dedup) %s stale data (last contact exceeded max_stale)", path)
goto START
}
// Re-enable stale queries if allowed
if d.config.MaxStale != 0 {
if *d.config.MaxStale > 0 {
allowStale = true
}
@@ -408,8 +402,8 @@ func (d *DedupManager) parseData(path string, raw []byte) {
func (d *DedupManager) attemptLock(client *consulapi.Client, session string, sessionCh chan struct{}, t *template.Template) {
defer d.wg.Done()
START:
log.Printf("[INFO] (dedup) attempting lock for template hash %s", t.HexMD5)
basePath := path.Join(d.config.Deduplicate.Prefix, t.HexMD5)
log.Printf("[INFO] (dedup) attempting lock for template hash %s", t.ID())
basePath := path.Join(*d.config.Prefix, t.ID())
lopts := &consulapi.LockOptions{
Key: path.Join(basePath, "lock"),
Session: session,

View File

@@ -0,0 +1,146 @@
package manager
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/pkg/errors"
)
type RenderInput struct {
Backup bool
Contents []byte
Dry bool
DryStream io.Writer
Path string
Perms os.FileMode
}
type RenderResult struct {
DidRender bool
WouldRender bool
}
// Render atomically renders a file contents to disk, returning a result of
// whether it would have rendered and actually did render.
func Render(i *RenderInput) (*RenderResult, error) {
existing, err := ioutil.ReadFile(i.Path)
if err != nil && !os.IsNotExist(err) {
return nil, errors.Wrap(err, "failed reading file")
}
if bytes.Equal(existing, i.Contents) {
return &RenderResult{
DidRender: false,
WouldRender: true,
}, nil
}
if i.Dry {
fmt.Fprintf(i.DryStream, "> %s\n%s", i.Path, i.Contents)
} else {
if err := AtomicWrite(i.Path, i.Contents, i.Perms, i.Backup); err != nil {
return nil, errors.Wrap(err, "failed writing file")
}
}
return &RenderResult{
DidRender: true,
WouldRender: true,
}, nil
}
// AtomicWrite accepts a destination path and the template contents. It writes
// the template contents to a TempFile on disk, returning if any errors occur.
//
// If the parent destination directory does not exist, it will be created
// automatically with permissions 0755. To use a different permission, create
// the directory first or use `chmod` in a Command.
//
// If the destination path exists, all attempts will be made to preserve the
// existing file permissions. If those permissions cannot be read, an error is
// returned. If the file does not exist, it will be created automatically with
// permissions 0644. To use a different permission, create the destination file
// first or use `chmod` in a Command.
//
// If no errors occur, the Tempfile is "renamed" (moved) to the destination
// path.
func AtomicWrite(path string, contents []byte, perms os.FileMode, backup bool) error {
if path == "" {
return fmt.Errorf("missing destination")
}
parent := filepath.Dir(path)
if _, err := os.Stat(parent); os.IsNotExist(err) {
if err := os.MkdirAll(parent, 0755); err != nil {
return err
}
}
f, err := ioutil.TempFile(parent, "")
if err != nil {
return err
}
defer os.Remove(f.Name())
if _, err := f.Write(contents); err != nil {
return err
}
if err := f.Sync(); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
if err := os.Chmod(f.Name(), perms); err != nil {
return err
}
// If we got this far, it means we are about to save the file. Copy the
// current contents of the file onto disk (if it exists) so we have a backup.
if backup {
if _, err := os.Stat(path); !os.IsNotExist(err) {
if err := copyFile(path, path+".bak"); err != nil {
return err
}
}
}
if err := os.Rename(f.Name(), path); err != nil {
return err
}
return nil
}
// copyFile copies the file at src to the path at dst. Any errors that occur
// are returned.
func copyFile(src, dst string) error {
s, err := os.Open(src)
if err != nil {
return err
}
defer s.Close()
stat, err := s.Stat()
if err != nil {
return err
}
d, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, stat.Mode())
if err != nil {
return err
}
if _, err := io.Copy(d, s); err != nil {
d.Close()
return err
}
return d.Close()
}

File diff suppressed because it is too large Load Diff

View File

@@ -7,13 +7,16 @@ import (
"strings"
)
// SIGNIL is the nil signal.
var SIGNIL os.Signal = new(NilSignal)
// ValidSignals is the list of all valid signals. This is built at runtime
// because it is OS-dependent.
var ValidSignals []string
func init() {
valid := make([]string, 0, len(SignalLookup))
for k, _ := range SignalLookup {
for k := range SignalLookup {
valid = append(valid, k)
}
sort.Strings(valid)
@@ -26,7 +29,7 @@ func Parse(s string) (os.Signal, error) {
sig, ok := SignalLookup[strings.ToUpper(s)]
if !ok {
return nil, fmt.Errorf("invalid signal %q - valid signals are %q",
sig, ValidSignals)
s, ValidSignals)
}
return sig, nil
}

View File

@@ -11,8 +11,8 @@ import (
type Brain struct {
sync.RWMutex
// data is the map of individual dependencies (by HashCode()) and the most
// recent data for that dependency.
// data is the map of individual dependencies and the most recent data for
// that dependency.
data map[string]interface{}
// receivedData is an internal tracker of which dependencies have stored data
@@ -36,8 +36,8 @@ func (b *Brain) Remember(d dep.Dependency, data interface{}) {
b.Lock()
defer b.Unlock()
b.data[d.HashCode()] = data
b.receivedData[d.HashCode()] = struct{}{}
b.data[d.String()] = data
b.receivedData[d.String()] = struct{}{}
}
// Recall gets the current value for the given dependency in the Brain.
@@ -46,11 +46,11 @@ func (b *Brain) Recall(d dep.Dependency) (interface{}, bool) {
defer b.RUnlock()
// If we have not received data for this dependency, return now.
if _, ok := b.receivedData[d.HashCode()]; !ok {
if _, ok := b.receivedData[d.String()]; !ok {
return nil, false
}
return b.data[d.HashCode()], true
return b.data[d.String()], true
}
// ForceSet is used to force set the value of a dependency
@@ -69,6 +69,6 @@ func (b *Brain) Forget(d dep.Dependency) {
b.Lock()
defer b.Unlock()
delete(b.data, d.HashCode())
delete(b.receivedData, d.HashCode())
delete(b.data, d.String())
delete(b.receivedData, d.String())
}

View File

@@ -2,20 +2,21 @@ package template
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"reflect"
"regexp"
"strconv"
"strings"
"text/template"
"time"
"github.com/burntsushi/toml"
dep "github.com/hashicorp/consul-template/dependency"
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v2"
)
@@ -24,105 +25,158 @@ import (
var now = func() time.Time { return time.Now().UTC() }
// datacentersFunc returns or accumulates datacenter dependencies.
func datacentersFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(...string) ([]string, error) {
return func(s ...string) ([]string, error) {
func datacentersFunc(b *Brain, used, missing *dep.Set) func() ([]string, error) {
return func() ([]string, error) {
result := []string{}
d, err := dep.ParseDatacenters(s...)
d, err := dep.NewCatalogDatacentersQuery()
if err != nil {
return result, err
}
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
return value.([]string), nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// envFunc returns a function which checks the value of an environment variable.
// Invokers can specify their own environment, which takes precedences over any
// real environment variables
func envFunc(b *Brain, used, missing *dep.Set, overrides []string) func(string) (string, error) {
return func(s string) (string, error) {
var result string
d, err := dep.NewEnvQuery(s)
if err != nil {
return result, err
}
used.Add(d)
// Overrides lookup - we have to do this after adding the dependency,
// otherwise dedupe sharing won't work.
for _, e := range overrides {
split := strings.SplitN(e, "=", 2)
k, v := split[0], split[1]
if k == s {
return v, nil
}
}
if value, ok := b.Recall(d); ok {
return value.(string), nil
}
missing.Add(d)
return result, nil
}
}
// executeTemplateFunc executes the given template in the context of the
// parent. If an argument is specified, it will be used as the context instead.
// This can be used for nested template definitions.
func executeTemplateFunc(t *template.Template) func(string, ...interface{}) (string, error) {
return func(s string, data ...interface{}) (string, error) {
var dot interface{}
switch len(data) {
case 0:
dot = nil
case 1:
dot = data[0]
default:
return "", fmt.Errorf("executeTemplate: wrong number of arguments, expected 1 or 2"+
", but got %d", len(data)+1)
}
var b bytes.Buffer
if err := t.ExecuteTemplate(&b, s, dot); err != nil {
return "", err
}
return b.String(), nil
}
}
// fileFunc returns or accumulates file dependencies.
func fileFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string) (string, error) {
func fileFunc(b *Brain, used, missing *dep.Set) func(string) (string, error) {
return func(s string) (string, error) {
if len(s) == 0 {
return "", nil
}
d, err := dep.ParseFile(s)
d, err := dep.NewFileQuery(s)
if err != nil {
return "", err
}
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
if value == nil {
return "", nil
}
return value.(string), nil
}
addDependency(missing, d)
missing.Add(d)
return "", nil
}
}
// keyFunc returns or accumulates key dependencies.
func keyFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string) (string, error) {
func keyFunc(b *Brain, used, missing *dep.Set) func(string) (string, error) {
return func(s string) (string, error) {
if len(s) == 0 {
return "", nil
}
d, err := dep.ParseStoreKey(s)
d, err := dep.NewKVGetQuery(s)
if err != nil {
return "", err
}
d.EnableBlocking()
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
if value == nil {
return "", nil
}
return value.(string), nil
}
addDependency(missing, d)
missing.Add(d)
return "", nil
}
}
// keyExistsFunc returns true if a key exists, false otherwise.
func keyExistsFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string) (bool, error) {
func keyExistsFunc(b *Brain, used, missing *dep.Set) func(string) (bool, error) {
return func(s string) (bool, error) {
if len(s) == 0 {
return false, nil
}
d, err := dep.ParseStoreKey(s)
d, err := dep.NewKVGetQuery(s)
if err != nil {
return false, err
}
d.SetExistenceCheck(true)
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
return value.(bool), nil
if value, ok := b.Recall(d); ok {
return value != nil, nil
}
addDependency(missing, d)
missing.Add(d)
return false, nil
}
@@ -130,37 +184,34 @@ func keyExistsFunc(brain *Brain,
// keyWithDefaultFunc returns or accumulates key dependencies that have a
// default value.
func keyWithDefaultFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string, string) (string, error) {
func keyWithDefaultFunc(b *Brain, used, missing *dep.Set) func(string, string) (string, error) {
return func(s, def string) (string, error) {
if len(s) == 0 {
return def, nil
}
d, err := dep.ParseStoreKey(s)
d, err := dep.NewKVGetQuery(s)
if err != nil {
return "", err
}
d.SetDefault(def)
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
if value == nil {
if value, ok := b.Recall(d); ok {
if value == nil || value.(string) == "" {
return def, nil
}
return value.(string), nil
}
addDependency(missing, d)
missing.Add(d)
return def, nil
}
}
// lsFunc returns or accumulates keyPrefix dependencies.
func lsFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string) ([]*dep.KeyPair, error) {
func lsFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, error) {
return func(s string) ([]*dep.KeyPair, error) {
result := []*dep.KeyPair{}
@@ -168,15 +219,15 @@ func lsFunc(brain *Brain,
return result, nil
}
d, err := dep.ParseStoreKeyPrefix(s)
d, err := dep.NewKVListQuery(s)
if err != nil {
return result, err
}
addDependency(used, d)
used.Add(d)
// Only return non-empty top-level keys
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
for _, pair := range value.([]*dep.KeyPair) {
if pair.Key != "" && !strings.Contains(pair.Key, "/") {
result = append(result, pair)
@@ -185,60 +236,57 @@ func lsFunc(brain *Brain,
return result, nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// nodeFunc returns or accumulates catalog node dependency.
func nodeFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(...string) (*dep.NodeDetail, error) {
return func(s ...string) (*dep.NodeDetail, error) {
func nodeFunc(b *Brain, used, missing *dep.Set) func(...string) (*dep.CatalogNode, error) {
return func(s ...string) (*dep.CatalogNode, error) {
d, err := dep.ParseCatalogNode(s...)
d, err := dep.NewCatalogNodeQuery(strings.Join(s, ""))
if err != nil {
return nil, err
}
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
return value.(*dep.NodeDetail), nil
if value, ok := b.Recall(d); ok {
return value.(*dep.CatalogNode), nil
}
addDependency(missing, d)
missing.Add(d)
return nil, nil
}
}
// nodesFunc returns or accumulates catalog node dependencies.
func nodesFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(...string) ([]*dep.Node, error) {
func nodesFunc(b *Brain, used, missing *dep.Set) func(...string) ([]*dep.Node, error) {
return func(s ...string) ([]*dep.Node, error) {
result := []*dep.Node{}
d, err := dep.ParseCatalogNodes(s...)
d, err := dep.NewCatalogNodesQuery(strings.Join(s, ""))
if err != nil {
return nil, err
}
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
return value.([]*dep.Node), nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// secretFunc returns or accumulates secret dependencies from Vault.
func secretFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(...string) (*dep.Secret, error) {
func secretFunc(b *Brain, used, missing *dep.Set) func(...string) (*dep.Secret, error) {
return func(s ...string) (*dep.Secret, error) {
result := &dep.Secret{}
@@ -246,27 +294,47 @@ func secretFunc(brain *Brain,
return result, nil
}
d, err := dep.ParseVaultSecret(s...)
if err != nil {
return result, nil
// TODO: Refactor into separate template functions
path, rest := s[0], s[1:]
data := make(map[string]interface{})
for _, str := range rest {
parts := strings.SplitN(str, "=", 2)
if len(parts) != 2 {
return result, nil
}
k, v := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])
data[k] = v
}
addDependency(used, d)
var d dep.Dependency
var err error
if value, ok := brain.Recall(d); ok {
if len(rest) == 0 {
d, err = dep.NewVaultReadQuery(path)
} else {
d, err = dep.NewVaultWriteQuery(path, data)
}
if err != nil {
return nil, err
}
used.Add(d)
if value, ok := b.Recall(d); ok {
result = value.(*dep.Secret)
return result, nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// secretsFunc returns or accumulates a list of secret dependencies from Vault.
func secretsFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string) ([]string, error) {
func secretsFunc(b *Brain, used, missing *dep.Set) func(string) ([]string, error) {
return func(s string) ([]string, error) {
result := []string{}
@@ -274,27 +342,26 @@ func secretsFunc(brain *Brain,
return result, nil
}
d, err := dep.ParseVaultSecrets(s)
d, err := dep.NewVaultListQuery(s)
if err != nil {
return result, nil
return nil, err
}
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
result = value.([]string)
return result, nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// serviceFunc returns or accumulates health service dependencies.
func serviceFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(...string) ([]*dep.HealthService, error) {
func serviceFunc(b *Brain, used, missing *dep.Set) func(...string) ([]*dep.HealthService, error) {
return func(s ...string) ([]*dep.HealthService, error) {
result := []*dep.HealthService{}
@@ -302,49 +369,47 @@ func serviceFunc(brain *Brain,
return result, nil
}
d, err := dep.ParseHealthServices(s...)
d, err := dep.NewHealthServiceQuery(strings.Join(s, ""))
if err != nil {
return nil, err
}
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
return value.([]*dep.HealthService), nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// servicesFunc returns or accumulates catalog services dependencies.
func servicesFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(...string) ([]*dep.CatalogService, error) {
return func(s ...string) ([]*dep.CatalogService, error) {
result := []*dep.CatalogService{}
func servicesFunc(b *Brain, used, missing *dep.Set) func(...string) ([]*dep.CatalogSnippet, error) {
return func(s ...string) ([]*dep.CatalogSnippet, error) {
result := []*dep.CatalogSnippet{}
d, err := dep.ParseCatalogServices(s...)
d, err := dep.NewCatalogServicesQuery(strings.Join(s, ""))
if err != nil {
return nil, err
}
addDependency(used, d)
used.Add(d)
if value, ok := brain.Recall(d); ok {
return value.([]*dep.CatalogService), nil
if value, ok := b.Recall(d); ok {
return value.([]*dep.CatalogSnippet), nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// treeFunc returns or accumulates keyPrefix dependencies.
func treeFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string) ([]*dep.KeyPair, error) {
func treeFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, error) {
return func(s string) ([]*dep.KeyPair, error) {
result := []*dep.KeyPair{}
@@ -352,15 +417,15 @@ func treeFunc(brain *Brain,
return result, nil
}
d, err := dep.ParseStoreKeyPrefix(s)
d, err := dep.NewKVListQuery(s)
if err != nil {
return result, err
}
addDependency(used, d)
used.Add(d)
// Only return non-empty top-level keys
if value, ok := brain.Recall(d); ok {
if value, ok := b.Recall(d); ok {
for _, pair := range value.([]*dep.KeyPair) {
parts := strings.Split(pair.Key, "/")
if parts[len(parts)-1] != "" {
@@ -370,20 +435,39 @@ func treeFunc(brain *Brain,
return result, nil
}
addDependency(missing, d)
missing.Add(d)
return result, nil
}
}
// vaultFunc is deprecated. Use secretFunc instead.
func vaultFunc(brain *Brain,
used, missing map[string]dep.Dependency) func(string) (*dep.Secret, error) {
return func(s string) (*dep.Secret, error) {
log.Printf("[WARN] the `vault' template function has been deprecated. " +
"Please use `secret` instead!")
return secretFunc(brain, used, missing)(s)
// base64Decode decodes the given string as a base64 string, returning an error
// if it fails.
func base64Decode(s string) (string, error) {
v, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return "", errors.Wrap(err, "base64Decode")
}
return string(v), nil
}
// base64Encode encodes the given value into a string represented as base64.
func base64Encode(s string) (string, error) {
return base64.StdEncoding.EncodeToString([]byte(s)), nil
}
// base64URLDecode decodes the given string as a URL-safe base64 string.
func base64URLDecode(s string) (string, error) {
v, err := base64.URLEncoding.DecodeString(s)
if err != nil {
return "", errors.Wrap(err, "base64URLDecode")
}
return string(v), nil
}
// base64URLEncode encodes the given string to be URL-safe.
func base64URLEncode(s string) (string, error) {
return base64.URLEncoding.EncodeToString([]byte(s)), nil
}
// byKey accepts a slice of KV pairs and returns a map of the top-level
@@ -436,12 +520,18 @@ func byTag(in interface{}) (map[string][]interface{}, error) {
switch typed := in.(type) {
case nil:
case []*dep.CatalogService:
case []*dep.CatalogSnippet:
for _, s := range typed {
for _, t := range s.Tags {
m[t] = append(m[t], s)
}
}
case []*dep.CatalogService:
for _, s := range typed {
for _, t := range s.ServiceTags {
m[t] = append(m[t], s)
}
}
case []*dep.HealthService:
for _, s := range typed {
for _, t := range s.Tags {
@@ -464,9 +554,24 @@ func contains(v, l interface{}) (bool, error) {
return in(l, v)
}
// env returns the value of the environment variable set
func env(s string) (string, error) {
return os.Getenv(s), nil
// containsSomeFunc returns functions to implement each of the following:
//
// 1. containsAll - true if (∀x ∈ v then x ∈ l); false otherwise
// 2. containsAny - true if (∃x ∈ v such that x ∈ l); false otherwise
// 3. containsNone - true if (∀x ∈ v then x ∉ l); false otherwise
// 2. containsNotAll - true if (∃x ∈ v such that x ∉ l); false otherwise
//
// ret_true - return true at end of loop for none/all; false for any/notall
// invert - invert block test for all/notall
func containsSomeFunc(retTrue, invert bool) func([]interface{}, interface{}) (bool, error) {
return func(v []interface{}, l interface{}) (bool, error) {
for i := 0; i < len(v); i++ {
if ok, _ := in(l, v[i]); ok != invert {
return !retTrue, nil
}
}
return retTrue, nil
}
}
// explode is used to expand a list of keypairs into a deeply-nested hash.
@@ -474,7 +579,7 @@ func explode(pairs []*dep.KeyPair) (map[string]interface{}, error) {
m := make(map[string]interface{})
for _, pair := range pairs {
if err := explodeHelper(m, pair.Key, pair.Value, pair.Key); err != nil {
return nil, err
return nil, errors.Wrap(err, "explode")
}
}
return m, nil
@@ -495,16 +600,16 @@ func explodeHelper(m map[string]interface{}, k, v, p string) error {
return fmt.Errorf("not a map: %q: %q already has value %q", p, top, m[top])
}
return explodeHelper(nest, key, v, k)
} else {
if k != "" {
m[k] = v
}
}
if k != "" {
m[k] = v
}
return nil
}
// in seaches for a given value in a given interface.
// in searches for a given value in a given interface.
func in(l, v interface{}) (bool, error) {
lv := reflect.ValueOf(l)
vv := reflect.ValueOf(v)
@@ -614,7 +719,7 @@ func parseBool(s string) (bool, error) {
result, err := strconv.ParseBool(s)
if err != nil {
return false, fmt.Errorf("parseBool: %s", err)
return false, errors.Wrap(err, "parseBool")
}
return result, nil
}
@@ -627,7 +732,7 @@ func parseFloat(s string) (float64, error) {
result, err := strconv.ParseFloat(s, 10)
if err != nil {
return 0, fmt.Errorf("parseFloat: %s", err)
return 0, errors.Wrap(err, "parseFloat")
}
return result, nil
}
@@ -640,7 +745,7 @@ func parseInt(s string) (int64, error) {
result, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return 0, fmt.Errorf("parseInt: %s", err)
return 0, errors.Wrap(err, "parseInt")
}
return result, nil
}
@@ -666,7 +771,7 @@ func parseUint(s string) (uint64, error) {
result, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return 0, fmt.Errorf("parseUint: %s", err)
return 0, errors.Wrap(err, "parseUint")
}
return result, nil
}
@@ -704,14 +809,14 @@ func plugin(name string, args ...string) (string, error) {
}()
select {
case <-time.After(5 * time.Second):
case <-time.After(30 * time.Second):
if cmd.Process != nil {
if err := cmd.Process.Kill(); err != nil {
return "", fmt.Errorf("exec %q: failed to kill", name)
}
}
<-done // Allow the goroutine to exit
return "", fmt.Errorf("exec %q: did not finish", name)
return "", fmt.Errorf("exec %q: did not finishin 30s", name)
case err := <-done:
if err != nil {
return "", fmt.Errorf("exec %q: %s\n\nstdout:\n\n%s\n\nstderr:\n\n%s",
@@ -783,7 +888,7 @@ func toLower(s string) (string, error) {
func toJSON(i interface{}) (string, error) {
result, err := json.Marshal(i)
if err != nil {
return "", fmt.Errorf("toJSON: %s", err)
return "", errors.Wrap(err, "toJSON")
}
return string(bytes.TrimSpace(result)), err
}
@@ -793,7 +898,7 @@ func toJSON(i interface{}) (string, error) {
func toJSONPretty(m map[string]interface{}) (string, error) {
result, err := json.MarshalIndent(m, "", " ")
if err != nil {
return "", fmt.Errorf("toJSONPretty: %s", err)
return "", errors.Wrap(err, "toJSONPretty")
}
return string(bytes.TrimSpace(result)), err
}
@@ -812,7 +917,7 @@ func toUpper(s string) (string, error) {
func toYAML(m map[string]interface{}) (string, error) {
result, err := yaml.Marshal(m)
if err != nil {
return "", fmt.Errorf("toYAML: %s", err)
return "", errors.Wrap(err, "toYAML")
}
return string(bytes.TrimSpace(result)), nil
}
@@ -822,11 +927,11 @@ func toTOML(m map[string]interface{}) (string, error) {
buf := bytes.NewBuffer([]byte{})
enc := toml.NewEncoder(buf)
if err := enc.Encode(m); err != nil {
return "", fmt.Errorf("toTOML: %s", err)
return "", errors.Wrap(err, "toTOML")
}
result, err := ioutil.ReadAll(buf)
if err != nil {
return "", fmt.Errorf("toTOML: %s", err)
return "", errors.Wrap(err, "toTOML")
}
return string(bytes.TrimSpace(result)), nil
}
@@ -1006,10 +1111,3 @@ func divide(b, a interface{}) (interface{}, error) {
return nil, fmt.Errorf("divide: unknown type for %q (%T)", av, a)
}
}
// addDependency adds the given Dependency to the map.
func addDependency(m map[string]dep.Dependency, d dep.Dependency) {
if _, ok := m[d.HashCode()]; !ok {
m[d.HashCode()] = d
}
}

View File

@@ -0,0 +1,125 @@
package template
import (
"fmt"
"sort"
"sync"
)
// Scratch is a wrapper around a map which is used by the template.
type Scratch struct {
once sync.Once
sync.RWMutex
values map[string]interface{}
}
// Key returns a boolean indicating whether the given key exists in the map.
func (s *Scratch) Key(k string) bool {
s.RLock()
defer s.RUnlock()
_, ok := s.values[k]
return ok
}
// Get returns a value previously set by Add or Set
func (s *Scratch) Get(k string) interface{} {
s.RLock()
defer s.RUnlock()
return s.values[k]
}
// Set stores the value v at the key k. It will overwrite an existing value
// if present.
func (s *Scratch) Set(k string, v interface{}) string {
s.init()
s.Lock()
defer s.Unlock()
s.values[k] = v
return ""
}
// SetX behaves the same as Set, except it will not overwrite existing keys if
// already present.
func (s *Scratch) SetX(k string, v interface{}) string {
s.init()
s.Lock()
defer s.Unlock()
if _, ok := s.values[k]; !ok {
s.values[k] = v
}
return ""
}
// MapSet stores the value v into a key mk in the map named k.
func (s *Scratch) MapSet(k, mk string, v interface{}) (string, error) {
s.init()
s.Lock()
defer s.Unlock()
return s.mapSet(k, mk, v, true)
}
// MapSetX behaves the same as MapSet, except it will not overwrite the map
// key if it already exists.
func (s *Scratch) MapSetX(k, mk string, v interface{}) (string, error) {
s.init()
s.Lock()
defer s.Unlock()
return s.mapSet(k, mk, v, false)
}
// mapSet is sets the value in the map, overwriting if o is true. This function
// does not perform locking; callers should lock before invoking.
func (s *Scratch) mapSet(k, mk string, v interface{}, o bool) (string, error) {
if _, ok := s.values[k]; !ok {
s.values[k] = make(map[string]interface{})
}
typed, ok := s.values[k].(map[string]interface{})
if !ok {
return "", fmt.Errorf("%q is not a map", k)
}
if _, ok := typed[mk]; o || !ok {
typed[mk] = v
}
return "", nil
}
// MapValues returns the list of values in the map sorted by key.
func (s *Scratch) MapValues(k string) ([]interface{}, error) {
s.init()
s.Lock()
defer s.Unlock()
if s.values == nil {
return nil, nil
}
typed, ok := s.values[k].(map[string]interface{})
if !ok {
return nil, fmt.Errorf("%q is not a map", k)
}
keys := make([]string, 0, len(typed))
for k := range typed {
keys = append(keys, k)
}
sort.Strings(keys)
sorted := make([]interface{}, len(keys))
for i, k := range keys {
sorted[i] = typed[k]
}
return sorted, nil
}
// init initializes the scratch.
func (s *Scratch) init() {
if s.values == nil {
s.values = make(map[string]interface{})
}
}

View File

@@ -4,144 +4,218 @@ import (
"bytes"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io/ioutil"
"path/filepath"
"text/template"
"github.com/pkg/errors"
dep "github.com/hashicorp/consul-template/dependency"
)
var (
// ErrTemplateContentsAndSource is the error returned when a template
// specifies both a "source" and "content" argument, which is not valid.
ErrTemplateContentsAndSource = errors.New("template: cannot specify both 'source' and 'content'")
// ErrTemplateMissingContentsAndSource is the error returned when a template
// does not specify either a "source" or "content" argument, which is not
// valid.
ErrTemplateMissingContentsAndSource = errors.New("template: must specify exactly one of 'source' or 'content'")
)
// Template is the internal representation of an individual template to process.
// The template retains the relationship between it's contents and is
// responsible for it's own execution.
type Template struct {
// Path is the path to this template on disk.
Path string
// LeftDelim and RightDelim are the left and right delimiters to use.
LeftDelim, RightDelim string
// Contents is the string contents for the template. It is either given
// contents is the string contents for the template. It is either given
// during template creation or read from disk when initialized.
contents string
// source is the original location of the template. This may be undefined if
// the template was dynamically defined.
source string
// leftDelim and rightDelim are the template delimiters.
leftDelim string
rightDelim string
// hexMD5 stores the hex version of the MD5
hexMD5 string
}
// NewTemplateInput is used as input when creating the template.
type NewTemplateInput struct {
// Source is the location on disk to the file.
Source string
// Contents are the raw template contents.
Contents string
// HexMD5 stores the hex version of the MD5
HexMD5 string
// LeftDelim and RightDelim are the template delimiters.
LeftDelim string
RightDelim string
}
// NewTemplate creates and parses a new Consul Template template at the given
// path. If the template does not exist, an error is returned. During
// initialization, the template is read and is parsed for dependencies. Any
// errors that occur are returned.
func NewTemplate(path, contents, leftDelim, rightDelim string) (*Template, error) {
func NewTemplate(i *NewTemplateInput) (*Template, error) {
if i == nil {
i = &NewTemplateInput{}
}
// Validate that we are either given the path or the explicit contents
pathEmpty, contentsEmpty := path == "", contents == ""
if !pathEmpty && !contentsEmpty {
return nil, errors.New("Either specify template path or content, not both")
} else if pathEmpty && contentsEmpty {
return nil, errors.New("Must specify template path or content")
if i.Source != "" && i.Contents != "" {
return nil, ErrTemplateContentsAndSource
} else if i.Source == "" && i.Contents == "" {
return nil, ErrTemplateMissingContentsAndSource
}
template := &Template{
Path: path,
Contents: contents,
LeftDelim: leftDelim,
RightDelim: rightDelim,
}
if err := template.init(); err != nil {
return nil, err
}
var t Template
t.source = i.Source
t.contents = i.Contents
t.leftDelim = i.LeftDelim
t.rightDelim = i.RightDelim
return template, nil
}
// ID returns an identifier for the template
func (t *Template) ID() string {
return t.HexMD5
}
// Execute evaluates this template in the context of the given brain.
//
// The first return value is the list of used dependencies.
// The second return value is the list of missing dependencies.
// The third return value is the rendered text.
// The fourth return value any error that occurs.
func (t *Template) Execute(brain *Brain) ([]dep.Dependency, []dep.Dependency, []byte, error) {
usedMap := make(map[string]dep.Dependency)
missingMap := make(map[string]dep.Dependency)
name := filepath.Base(t.Path)
funcs := funcMap(brain, usedMap, missingMap)
tmpl, err := template.New(name).
Delims(t.LeftDelim, t.RightDelim).
Funcs(funcs).
Parse(t.Contents)
if err != nil {
return nil, nil, nil, fmt.Errorf("template: %s", err)
}
// TODO: accept an io.Writer instead
buff := new(bytes.Buffer)
if err := tmpl.Execute(buff, nil); err != nil {
return nil, nil, nil, fmt.Errorf("template: %s", err)
}
// Update this list of this template's dependencies
var used []dep.Dependency
for _, dep := range usedMap {
used = append(used, dep)
}
// Compile the list of missing dependencies
var missing []dep.Dependency
for _, dep := range missingMap {
missing = append(missing, dep)
}
return used, missing, buff.Bytes(), nil
}
// init reads the template file and initializes required variables.
func (t *Template) init() error {
// Render the template
if t.Path != "" {
contents, err := ioutil.ReadFile(t.Path)
if i.Source != "" {
contents, err := ioutil.ReadFile(i.Source)
if err != nil {
return err
return nil, errors.Wrap(err, "failed to read template")
}
t.Contents = string(contents)
t.contents = string(contents)
}
// Compute the MD5, encode as hex
hash := md5.Sum([]byte(t.Contents))
t.HexMD5 = hex.EncodeToString(hash[:])
hash := md5.Sum([]byte(t.contents))
t.hexMD5 = hex.EncodeToString(hash[:])
return nil
return &t, nil
}
// ID returns the identifier for this template.
func (t *Template) ID() string {
return t.hexMD5
}
// Contents returns the raw contents of the template.
func (t *Template) Contents() string {
return t.contents
}
// Source returns the filepath source of this template.
func (t *Template) Source() string {
if t.source == "" {
return "(dynamic)"
}
return t.source
}
// ExecuteInput is used as input to the template's execute function.
type ExecuteInput struct {
// Brain is the brain where data for the template is stored.
Brain *Brain
// Env is a custom environment provided to the template for envvar resolution.
// Values specified here will take precedence over any values in the
// environment when using the `env` function.
Env []string
}
// ExecuteResult is the result of the template execution.
type ExecuteResult struct {
// Used is the set of dependencies that were used.
Used *dep.Set
// Missing is the set of dependencies that were missing.
Missing *dep.Set
// Output is the rendered result.
Output []byte
}
// Execute evaluates this template in the provided context.
func (t *Template) Execute(i *ExecuteInput) (*ExecuteResult, error) {
if i == nil {
i = &ExecuteInput{}
}
var used, missing dep.Set
tmpl := template.New("")
tmpl.Delims(t.leftDelim, t.rightDelim)
tmpl.Funcs(funcMap(&funcMapInput{
t: tmpl,
brain: i.Brain,
env: i.Env,
used: &used,
missing: &missing,
}))
tmpl, err := tmpl.Parse(t.contents)
if err != nil {
return nil, errors.Wrap(err, "parse")
}
// Execute the template into the writer
var b bytes.Buffer
if err := tmpl.Execute(&b, nil); err != nil {
return nil, errors.Wrap(err, "execute")
}
return &ExecuteResult{
Used: &used,
Missing: &missing,
Output: b.Bytes(),
}, nil
}
// funcMapInput is input to the funcMap, which builds the template functions.
type funcMapInput struct {
t *template.Template
brain *Brain
env []string
used *dep.Set
missing *dep.Set
}
// funcMap is the map of template functions to their respective functions.
func funcMap(brain *Brain, used, missing map[string]dep.Dependency) template.FuncMap {
func funcMap(i *funcMapInput) template.FuncMap {
var scratch Scratch
return template.FuncMap{
// API functions
"datacenters": datacentersFunc(brain, used, missing),
"file": fileFunc(brain, used, missing),
"key": keyFunc(brain, used, missing),
"key_exists": keyExistsFunc(brain, used, missing),
"key_or_default": keyWithDefaultFunc(brain, used, missing),
"ls": lsFunc(brain, used, missing),
"node": nodeFunc(brain, used, missing),
"nodes": nodesFunc(brain, used, missing),
"secret": secretFunc(brain, used, missing),
"secrets": secretsFunc(brain, used, missing),
"service": serviceFunc(brain, used, missing),
"services": servicesFunc(brain, used, missing),
"tree": treeFunc(brain, used, missing),
"vault": vaultFunc(brain, used, missing),
"datacenters": datacentersFunc(i.brain, i.used, i.missing),
"env": envFunc(i.brain, i.used, i.missing, i.env),
"file": fileFunc(i.brain, i.used, i.missing),
"key": keyFunc(i.brain, i.used, i.missing),
"keyExists": keyExistsFunc(i.brain, i.used, i.missing),
"keyOrDefault": keyWithDefaultFunc(i.brain, i.used, i.missing),
"ls": lsFunc(i.brain, i.used, i.missing),
"node": nodeFunc(i.brain, i.used, i.missing),
"nodes": nodesFunc(i.brain, i.used, i.missing),
"secret": secretFunc(i.brain, i.used, i.missing),
"secrets": secretsFunc(i.brain, i.used, i.missing),
"service": serviceFunc(i.brain, i.used, i.missing),
"services": servicesFunc(i.brain, i.used, i.missing),
"tree": treeFunc(i.brain, i.used, i.missing),
// Scratch
"scratch": func() *Scratch { return &scratch },
// Helper functions
"base64Decode": base64Decode,
"base64Encode": base64Encode,
"base64URLDecode": base64URLDecode,
"base64URLEncode": base64URLEncode,
"byKey": byKey,
"byTag": byTag,
"contains": contains,
"env": env,
"containsAll": containsSomeFunc(true, true),
"containsAny": containsSomeFunc(false, false),
"containsNone": containsSomeFunc(true, false),
"containsNotAll": containsSomeFunc(false, true),
"executeTemplate": executeTemplateFunc(i.t),
"explode": explode,
"in": in,
"loop": loop,
@@ -171,5 +245,8 @@ func funcMap(brain *Brain, used, missing map[string]dep.Dependency) template.Fun
"subtract": subtract,
"multiply": multiply,
"divide": divide,
// Deprecated functions
"key_or_default": keyWithDefaultFunc(i.brain, i.used, i.missing),
}
}

View File

@@ -1,27 +0,0 @@
package watch
import (
"reflect"
"github.com/mitchellh/mapstructure"
)
// StringToWaitDurationHookFunc returns a function that converts strings to wait
// value. This is designed to be used with mapstructure for parsing out a wait
// value.
func StringToWaitDurationHookFunc() mapstructure.DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(new(Wait)) {
return data, nil
}
// Convert it by parsing
return ParseWait(data.(string))
}
}

View File

@@ -18,42 +18,72 @@ const (
// View is a representation of a Dependency and the most recent data it has
// received from Consul.
type View struct {
// Dependency is the dependency that is associated with this View
Dependency dep.Dependency
// dependency is the dependency that is associated with this View
dependency dep.Dependency
// config is the configuration for the watcher that created this view and
// contains important information about how this view should behave when
// polling including retry functions and handling stale queries.
config *WatcherConfig
// clients is the list of clients to communicate upstream. This is passed
// directly to the dependency.
clients *dep.ClientSet
// Data is the most-recently-received data from Consul for this View
// data is the most-recently-received data from Consul for this View. It is
// accompanied by a series of locks and booleans to ensure consistency.
dataLock sync.RWMutex
data interface{}
receivedData bool
lastIndex uint64
// maxStale is the maximum amount of time to allow a query to be stale.
maxStale time.Duration
// once determines if this view should receive data exactly once.
once bool
// retryFunc is the function to invoke on failure to determine if a retry
// should be attempted.
retryFunc RetryFunc
// stopCh is used to stop polling on this View
stopCh chan struct{}
}
// NewView creates a new view object from the given Consul API client and
// Dependency. If an error occurs, it will be returned.
func NewView(config *WatcherConfig, d dep.Dependency) (*View, error) {
if config == nil {
return nil, fmt.Errorf("view: missing config")
}
// NewViewInput is used as input to the NewView function.
type NewViewInput struct {
// Dependency is the dependency to associate with the new view.
Dependency dep.Dependency
if d == nil {
return nil, fmt.Errorf("view: missing dependency")
}
// Clients is the list of clients to communicate upstream. This is passed
// directly to the dependency.
Clients *dep.ClientSet
// MaxStale is the maximum amount a time a query response is allowed to be
// stale before forcing a read from the leader.
MaxStale time.Duration
// Once indicates this view should poll for data exactly one time.
Once bool
// RetryFunc is a function which dictates how this view should retry on
// upstream errors.
RetryFunc RetryFunc
}
// NewView constructs a new view with the given inputs.
func NewView(i *NewViewInput) (*View, error) {
return &View{
Dependency: d,
config: config,
stopCh: make(chan struct{}),
dependency: i.Dependency,
clients: i.Clients,
maxStale: i.MaxStale,
once: i.Once,
retryFunc: i.RetryFunc,
stopCh: make(chan struct{}, 1),
}, nil
}
// Dependency returns the dependency attached to this View.
func (v *View) Dependency() dep.Dependency {
return v.dependency
}
// Data returns the most-recently-received data from Consul for this View.
func (v *View) Data() interface{} {
v.dataLock.RLock()
@@ -75,8 +105,7 @@ func (v *View) DataAndLastIndex() (interface{}, uint64) {
// function to be fired in a goroutine, but then halted even if the fetch
// function is in the middle of a blocking query.
func (v *View) poll(viewCh chan<- *View, errCh chan<- error) {
defaultRetry := v.config.RetryFunc(1 * time.Second)
currentRetry := defaultRetry
var retries int
for {
doneCh, fetchErrCh := make(chan struct{}, 1), make(chan error, 1)
@@ -86,37 +115,47 @@ func (v *View) poll(viewCh chan<- *View, errCh chan<- error) {
case <-doneCh:
// Reset the retry to avoid exponentially incrementing retries when we
// have some successful requests
currentRetry = defaultRetry
retries = 0
log.Printf("[INFO] (view) %s received data", v.display())
log.Printf("[TRACE] (view) %s received data", v.dependency)
select {
case <-v.stopCh:
return
case viewCh <- v:
}
// If we are operating in once mode, do not loop - we received data at
// least once which is the API promise here.
if v.config.Once {
if v.once {
return
}
case err := <-fetchErrCh:
log.Printf("[ERR] (view) %s %s", v.display(), err)
if v.retryFunc != nil {
retry, sleep := v.retryFunc(retries)
if retry {
log.Printf("[WARN] (view) %s (retry attempt %d after %q)",
err, retries+1, sleep)
select {
case <-time.After(sleep):
retries++
continue
case <-v.stopCh:
return
}
}
}
log.Printf("[ERR] (view) %s (exceeded maximum retries)", err)
// Push the error back up to the watcher
select {
case <-v.stopCh:
return
case errCh <- err:
return
}
// Sleep and retry
if v.config.RetryFunc != nil {
currentRetry = v.config.RetryFunc(currentRetry)
}
log.Printf("[INFO] (view) %s errored, retrying in %s", v.display(), currentRetry)
time.Sleep(currentRetry)
continue
case <-v.stopCh:
log.Printf("[DEBUG] (view) %s stopping poll (received on view stopCh)", v.display())
log.Printf("[TRACE] (view) %s stopping poll (received on view stopCh)", v.dependency)
return
}
}
@@ -128,10 +167,10 @@ func (v *View) poll(viewCh chan<- *View, errCh chan<- error) {
// result of doneCh and errCh. It is assumed that only one instance of fetch
// is running per View and therefore no locking or mutexes are used.
func (v *View) fetch(doneCh chan<- struct{}, errCh chan<- error) {
log.Printf("[DEBUG] (view) %s starting fetch", v.display())
log.Printf("[TRACE] (view) %s starting fetch", v.dependency)
var allowStale bool
if v.config.MaxStale != 0 {
if v.maxStale != 0 {
allowStale = true
}
@@ -144,48 +183,44 @@ func (v *View) fetch(doneCh chan<- struct{}, errCh chan<- error) {
default:
}
opts := &dep.QueryOptions{
data, rm, err := v.dependency.Fetch(v.clients, &dep.QueryOptions{
AllowStale: allowStale,
WaitTime: defaultWaitTime,
WaitIndex: v.lastIndex,
}
data, rm, err := v.Dependency.Fetch(v.config.Clients, opts)
})
if err != nil {
// ErrStopped is returned by a dependency when it prematurely stopped
// because the upstream process asked for a reload or termination. The
// most likely cause is that the view was stopped due to a configuration
// reload or process interrupt, so we do not want to propagate this error
// to the runner, but we want to stop the fetch routine for this view.
if err != dep.ErrStopped {
if err == dep.ErrStopped {
log.Printf("[TRACE] (view) %s reported stop", v.dependency)
} else {
errCh <- err
}
return
}
if rm == nil {
errCh <- fmt.Errorf("consul returned nil response metadata; this " +
"should never happen and is probably a bug in consul-template")
errCh <- fmt.Errorf("received nil response metadata - this is a bug " +
"and should be reported")
return
}
if allowStale && rm.LastContact > v.config.MaxStale {
if allowStale && rm.LastContact > v.maxStale {
allowStale = false
log.Printf("[DEBUG] (view) %s stale data (last contact exceeded max_stale)", v.display())
log.Printf("[TRACE] (view) %s stale data (last contact exceeded max_stale)", v.dependency)
continue
}
if v.config.MaxStale != 0 {
if v.maxStale != 0 {
allowStale = true
}
if rm.LastIndex == v.lastIndex {
log.Printf("[DEBUG] (view) %s no new data (index was the same)", v.display())
log.Printf("[TRACE] (view) %s no new data (index was the same)", v.dependency)
continue
}
v.dataLock.Lock()
if rm.LastIndex < v.lastIndex {
log.Printf("[DEBUG] (view) %s had a lower index, resetting", v.display())
log.Printf("[TRACE] (view) %s had a lower index, resetting", v.dependency)
v.lastIndex = 0
v.dataLock.Unlock()
continue
@@ -193,13 +228,13 @@ func (v *View) fetch(doneCh chan<- struct{}, errCh chan<- error) {
v.lastIndex = rm.LastIndex
if v.receivedData && reflect.DeepEqual(data, v.data) {
log.Printf("[DEBUG] (view) %s no new data (contents were the same)", v.display())
log.Printf("[TRACE] (view) %s no new data (contents were the same)", v.dependency)
v.dataLock.Unlock()
continue
}
if data == nil {
log.Printf("[DEBUG](view) %s data was not present", v.display())
if data == nil && rm.Block {
log.Printf("[TRACE] (view) %s asked for blocking query", v.dependency)
v.dataLock.Unlock()
continue
}
@@ -213,13 +248,8 @@ func (v *View) fetch(doneCh chan<- struct{}, errCh chan<- error) {
}
}
// display returns a string that represents this view.
func (v *View) display() string {
return v.Dependency.Display()
}
// stop halts polling of this view.
func (v *View) stop() {
v.Dependency.Stop()
v.dependency.Stop()
close(v.stopCh)
}

View File

@@ -1,87 +0,0 @@
package watch
import (
"errors"
"fmt"
"strings"
"time"
)
// Wait is the Min/Max duration used by the Watcher
type Wait struct {
// Min and Max are the minimum and maximum time, respectively, to wait for
// data changes before rendering a new template to disk.
Min time.Duration `json:"min" mapstructure:"min"`
Max time.Duration `json:"max" mapstructure:"max"`
}
// ParseWait parses a string of the format `minimum(:maximum)` into a Wait
// struct.
func ParseWait(s string) (*Wait, error) {
if len(strings.TrimSpace(s)) < 1 {
return nil, errors.New("cannot specify empty wait interval")
}
parts := strings.Split(s, ":")
var min, max time.Duration
var err error
if len(parts) == 1 {
min, err = time.ParseDuration(strings.TrimSpace(parts[0]))
if err != nil {
return nil, err
}
max = 4 * min
} else if len(parts) == 2 {
min, err = time.ParseDuration(strings.TrimSpace(parts[0]))
if err != nil {
return nil, err
}
max, err = time.ParseDuration(strings.TrimSpace(parts[1]))
if err != nil {
return nil, err
}
} else {
return nil, errors.New("invalid wait interval format")
}
if min < 0 || max < 0 {
return nil, errors.New("cannot specify a negative wait interval")
}
if max < min {
return nil, errors.New("wait interval max must be larger than min")
}
return &Wait{min, max}, nil
}
// IsActive returns true if this wait is active (non-zero).
func (w *Wait) IsActive() bool {
return w.Min != 0 && w.Max != 0
}
// WaitVar implements the Flag.Value interface and allows the user to specify
// a watch interval using Go's flag parsing library.
type WaitVar Wait
// Set sets the value in the format min[:max] for a wait timer.
func (w *WaitVar) Set(value string) error {
wait, err := ParseWait(value)
if err != nil {
return err
}
w.Min = wait.Min
w.Max = wait.Max
return nil
}
// String returns the string format for this wait variable
func (w *WaitVar) String() string {
return fmt.Sprintf("%s:%s", w.Min, w.Max)
}

View File

@@ -1,77 +1,105 @@
package watch
import (
"fmt"
"log"
"sync"
"time"
dep "github.com/hashicorp/consul-template/dependency"
"github.com/pkg/errors"
)
// RetryFunc is a function that defines the retry for a given watcher. The
// function parameter is the current retry (which might be nil), and the
// return value is the new retry. In this way, you can build complex retry
// functions that are based off the previous values.
type RetryFunc func(time.Duration) time.Duration
// DefaultRetryFunc is the default return function, which just echos whatever
// duration it was given.
var DefaultRetryFunc RetryFunc = func(t time.Duration) time.Duration {
return t
}
// dataBufferSize is the default number of views to process in a batch.
const dataBufferSize = 2048
type RetryFunc func(int) (bool, time.Duration)
// Watcher is a top-level manager for views that poll Consul for data.
type Watcher struct {
sync.Mutex
// DataCh is the chan where Views will be published.
DataCh chan *View
// clients is the collection of API clients to talk to upstreams.
clients *dep.ClientSet
// ErrCh is the chan where any errors will be published.
ErrCh chan error
// dataCh is the chan where Views will be published.
dataCh chan *View
// config is the internal configuration of this watcher.
config *WatcherConfig
// errCh is the chan where any errors will be published.
errCh chan error
// depViewMap is a map of Templates to Views. Templates are keyed by
// HashCode().
// their string.
depViewMap map[string]*View
// maxStale specifies the maximum staleness of a query response.
maxStale time.Duration
// once signals if this watcher should tell views to retrieve data exactly
// one time intead of polling infinitely.
once bool
// retryFuncs specifies the different ways to retry based on the upstream.
retryFuncConsul RetryFunc
retryFuncDefault RetryFunc
retryFuncVault RetryFunc
}
// WatcherConfig is the configuration for a particular Watcher.
type WatcherConfig struct {
// Client is the mechanism for communicating with the Consul API.
type NewWatcherInput struct {
// Clients is the client set to communicate with upstreams.
Clients *dep.ClientSet
// Once is used to determine if the views should poll for data exactly once.
Once bool
// MaxStale is the maximum staleness of a query. If specified, Consul will
// distribute work among all servers instead of just the leader. Specifying
// this option assumes the use of AllowStale.
// MaxStale is the maximum staleness of a query.
MaxStale time.Duration
// RetryFunc is a RetryFunc that represents the way retrys and backoffs
// should occur.
RetryFunc RetryFunc
// Once specifies this watcher should tell views to poll exactly once.
Once bool
// RenewVault determines if the watcher should renew the Vault token as a
// background job.
// RenewVault indicates if this watcher should renew Vault tokens.
RenewVault bool
// RetryFuncs specify the different ways to retry based on the upstream.
RetryFuncConsul RetryFunc
RetryFuncDefault RetryFunc
RetryFuncVault RetryFunc
}
// NewWatcher creates a new watcher using the given API client.
func NewWatcher(config *WatcherConfig) (*Watcher, error) {
watcher := &Watcher{config: config}
if err := watcher.init(); err != nil {
return nil, err
func NewWatcher(i *NewWatcherInput) (*Watcher, error) {
w := &Watcher{
clients: i.Clients,
depViewMap: make(map[string]*View),
dataCh: make(chan *View, dataBufferSize),
errCh: make(chan error),
maxStale: i.MaxStale,
once: i.Once,
retryFuncConsul: i.RetryFuncConsul,
retryFuncDefault: i.RetryFuncDefault,
retryFuncVault: i.RetryFuncVault,
}
return watcher, nil
// Start a watcher for the Vault renew if that config was specified
if i.RenewVault {
vt, err := dep.NewVaultTokenQuery()
if err != nil {
return nil, errors.Wrap(err, "watcher")
}
if _, err := w.Add(vt); err != nil {
return nil, errors.Wrap(err, "watcher")
}
}
return w, nil
}
// DataCh returns a read-only channel of Views which is populated when a view
// receives data from its upstream.
func (w *Watcher) DataCh() <-chan *View {
return w.dataCh
}
// ErrCh returns a read-only channel of errors returned by the upstream.
func (w *Watcher) ErrCh() <-chan error {
return w.errCh
}
// Add adds the given dependency to the list of monitored depedencies
@@ -86,22 +114,39 @@ func (w *Watcher) Add(d dep.Dependency) (bool, error) {
w.Lock()
defer w.Unlock()
log.Printf("[INFO] (watcher) adding %s", d.Display())
log.Printf("[DEBUG] (watcher) adding %s", d)
if _, ok := w.depViewMap[d.HashCode()]; ok {
log.Printf("[DEBUG] (watcher) %s already exists, skipping", d.Display())
if _, ok := w.depViewMap[d.String()]; ok {
log.Printf("[TRACE] (watcher) %s already exists, skipping", d)
return false, nil
}
v, err := NewView(w.config, d)
if err != nil {
return false, err
// Choose the correct retry function based off of the dependency's type.
var retryFunc RetryFunc
switch d.Type() {
case dep.TypeConsul:
retryFunc = w.retryFuncConsul
case dep.TypeVault:
retryFunc = w.retryFuncVault
default:
retryFunc = w.retryFuncDefault
}
log.Printf("[DEBUG] (watcher) %s starting", d.Display())
v, err := NewView(&NewViewInput{
Dependency: d,
Clients: w.clients,
MaxStale: w.maxStale,
Once: w.once,
RetryFunc: retryFunc,
})
if err != nil {
return false, errors.Wrap(err, "watcher")
}
w.depViewMap[d.HashCode()] = v
go v.poll(w.DataCh, w.ErrCh)
log.Printf("[TRACE] (watcher) %s starting", d)
w.depViewMap[d.String()] = v
go v.poll(w.dataCh, w.errCh)
return true, nil
}
@@ -111,7 +156,7 @@ func (w *Watcher) Watching(d dep.Dependency) bool {
w.Lock()
defer w.Unlock()
_, ok := w.depViewMap[d.HashCode()]
_, ok := w.depViewMap[d.String()]
return ok
}
@@ -122,9 +167,9 @@ func (w *Watcher) ForceWatching(d dep.Dependency, enabled bool) {
defer w.Unlock()
if enabled {
w.depViewMap[d.HashCode()] = nil
w.depViewMap[d.String()] = nil
} else {
delete(w.depViewMap, d.HashCode())
delete(w.depViewMap, d.String())
}
}
@@ -136,16 +181,16 @@ func (w *Watcher) Remove(d dep.Dependency) bool {
w.Lock()
defer w.Unlock()
log.Printf("[INFO] (watcher) removing %s", d.Display())
log.Printf("[DEBUG] (watcher) removing %s", d)
if view, ok := w.depViewMap[d.HashCode()]; ok {
log.Printf("[DEBUG] (watcher) actually removing %s", d.Display())
if view, ok := w.depViewMap[d.String()]; ok {
log.Printf("[TRACE] (watcher) actually removing %s", d)
view.stop()
delete(w.depViewMap, d.HashCode())
delete(w.depViewMap, d.String())
return true
}
log.Printf("[DEBUG] (watcher) %s did not exist, skipping", d.Display())
log.Printf("[TRACE] (watcher) %s did not exist, skipping", d)
return false
}
@@ -162,13 +207,13 @@ func (w *Watcher) Stop() {
w.Lock()
defer w.Unlock()
log.Printf("[INFO] (watcher) stopping all views")
log.Printf("[DEBUG] (watcher) stopping all views")
for _, view := range w.depViewMap {
if view == nil {
continue
}
log.Printf("[DEBUG] (watcher) stopping %s", view.Dependency.Display())
log.Printf("[TRACE] (watcher) stopping %s", view.Dependency())
view.stop()
}
@@ -176,36 +221,5 @@ func (w *Watcher) Stop() {
w.depViewMap = make(map[string]*View)
// Close any idle TCP connections
w.config.Clients.Stop()
}
// init sets up the initial values for the watcher.
func (w *Watcher) init() error {
if w.config == nil {
return fmt.Errorf("watcher: missing config")
}
if w.config.RetryFunc == nil {
w.config.RetryFunc = DefaultRetryFunc
}
// Setup the channels
w.DataCh = make(chan *View, dataBufferSize)
w.ErrCh = make(chan error)
// Setup our map of dependencies to views
w.depViewMap = make(map[string]*View)
// Start a watcher for the Vault renew if that config was specified
if w.config.RenewVault {
vt, err := dep.ParseVaultToken()
if err != nil {
return fmt.Errorf("watcher: %s", err)
}
if _, err := w.Add(vt); err != nil {
return fmt.Errorf("watcher: %s", err)
}
}
return nil
w.clients.Stop()
}

42
vendor/vendor.json vendored
View File

@@ -501,46 +501,46 @@
"revision": "a557574d6c024ed6e36acc8b610f5f211c91568a"
},
{
"checksumSHA1": "+JUQvWp1JUVeRT5weWL9hi6Fu4Y=",
"checksumSHA1": "gx2CAg/v3k7kfBA/rT5NCkI0jDI=",
"path": "github.com/hashicorp/consul-template/child",
"revision": "17cd016cdfa6601e82256b8d624b1331a0c188a7",
"revisionTime": "2016-10-28T21:56:23Z"
"revision": "81e8f468004caef5755e1073fe39ff48203a89b6",
"revisionTime": "2017-01-13T23:20:51Z"
},
{
"checksumSHA1": "UerCY17HM5DSJ/rE760qxm99Al4=",
"checksumSHA1": "1EAiHEm1b/m3KWy08U7yrAvDlms=",
"path": "github.com/hashicorp/consul-template/config",
"revision": "17cd016cdfa6601e82256b8d624b1331a0c188a7",
"revisionTime": "2016-10-28T21:56:23Z"
"revision": "81e8f468004caef5755e1073fe39ff48203a89b6",
"revisionTime": "2017-01-13T23:20:51Z"
},
{
"checksumSHA1": "0nA6tnACi/MkE+Mb5L1gqbc3tpw=",
"checksumSHA1": "1ioVPkGIVei92yBS2M07A3rBxPI=",
"path": "github.com/hashicorp/consul-template/dependency",
"revision": "17cd016cdfa6601e82256b8d624b1331a0c188a7",
"revisionTime": "2016-10-28T21:56:23Z"
"revision": "81e8f468004caef5755e1073fe39ff48203a89b6",
"revisionTime": "2017-01-13T23:20:51Z"
},
{
"checksumSHA1": "KcDxr/mNzYzTeFSCQyhpU1Nm/Ug=",
"checksumSHA1": "HnUdBWhBl8HicYR3f9ih1edtpq4=",
"path": "github.com/hashicorp/consul-template/manager",
"revision": "17cd016cdfa6601e82256b8d624b1331a0c188a7",
"revisionTime": "2016-10-28T21:56:23Z"
"revision": "81e8f468004caef5755e1073fe39ff48203a89b6",
"revisionTime": "2017-01-13T23:20:51Z"
},
{
"checksumSHA1": "ByMIKPf7bXpyhhy80IjKLKYrjpo=",
"checksumSHA1": "oskgb0WteBKOItG8NNDduM7E/D0=",
"path": "github.com/hashicorp/consul-template/signals",
"revision": "17cd016cdfa6601e82256b8d624b1331a0c188a7",
"revisionTime": "2016-10-28T21:56:23Z"
"revision": "81e8f468004caef5755e1073fe39ff48203a89b6",
"revisionTime": "2017-01-13T23:20:51Z"
},
{
"checksumSHA1": "bkSJRnR2VyZA1KoyOF/eSkxVVFg=",
"checksumSHA1": "0pGBuG4JrLmXMoXYahW9cH++lOA=",
"path": "github.com/hashicorp/consul-template/template",
"revision": "17cd016cdfa6601e82256b8d624b1331a0c188a7",
"revisionTime": "2016-10-28T21:56:23Z"
"revision": "81e8f468004caef5755e1073fe39ff48203a89b6",
"revisionTime": "2017-01-13T23:20:51Z"
},
{
"checksumSHA1": "HfWf4Vf1fBJh5HgHLdjpF5vs0Lk=",
"checksumSHA1": "cl9R28+I+YT6a0Z+KQFP//wuC+0=",
"path": "github.com/hashicorp/consul-template/watch",
"revision": "17cd016cdfa6601e82256b8d624b1331a0c188a7",
"revisionTime": "2016-10-28T21:56:23Z"
"revision": "81e8f468004caef5755e1073fe39ff48203a89b6",
"revisionTime": "2017-01-13T23:20:51Z"
},
{
"checksumSHA1": "kWbL0V4o8vJL75mzeQzhF6p5jiQ=",