diff --git a/client/fingerprint/network.go b/client/fingerprint/network.go new file mode 100644 index 000000000..5e3351d45 --- /dev/null +++ b/client/fingerprint/network.go @@ -0,0 +1,71 @@ +package fingerprint + +import ( + "io/ioutil" + "log" + "net/http" + "os" + "regexp" + "time" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" +) + +type NetworkFingerPrinter interface { + // Fingerprint collects information about the nodes network configuration + Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) + + // Interfaces returns a slice of connected interface devices for the node + Interfaces() []string + + // LinkSpeed queries a given interface device and returns speed information, + // in MB/s + LinkSpeed(device string) string +} + +func NetworkDefault(logger *log.Logger) NetworkFingerPrinter { + if isAWS() { + return NewAWSNetworkFingerprinter(logger) + } + return NewNetworkFingerprinter(logger) +} + +// isAWS queries the internal AWS Instance Metadata url, and determines if the +// node is running on AWS or not. +// TODO: Generalize this and use in other AWS related Fingerprinters +func isAWS() bool { + // Read the internal metadata URL from the environment, allowing test files to + // provide their own + metadataURL := os.Getenv("AWS_ENV_URL") + if metadataURL == "" { + metadataURL = "http://169.254.169.254/latest/meta-data/" + } + + // assume 2 seconds is enough time for inside AWS network + client := &http.Client{ + Timeout: 2 * time.Second, + } + + // Query the metadata url for the ami-id, to veryify we're on AWS + resp, err := client.Get(metadataURL + "ami-id") + + if err != nil { + log.Printf("[Err] Error querying AWS Metadata URL, skipping") + return false + } + defer resp.Body.Close() + + instanceID, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Printf("[Err] Error reading AWS Instance ID, skipping") + return false + } + + match, err := regexp.MatchString("ami-*", string(instanceID)) + if !match { + return false + } + + return true +} diff --git a/client/fingerprint/network_aws.go b/client/fingerprint/network_aws.go new file mode 100644 index 000000000..b1aee175b --- /dev/null +++ b/client/fingerprint/network_aws.go @@ -0,0 +1,150 @@ +// +build linux,darwin +package fingerprint + +import ( + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" +) + +// AWSNetworkFingerprint is used to fingerprint the Network capabilities of a node +type AWSNetworkFingerprint struct { + logger *log.Logger +} + +// AWSNetworkFingerprint is used to create a new AWS Network Fingerprinter +func NewAWSNetworkFingerprinter(logger *log.Logger) NetworkFingerPrinter { + f := &AWSNetworkFingerprint{logger: logger} + return f +} + +func (f *AWSNetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { + metadataURL := os.Getenv("AWS_ENV_URL") + if metadataURL == "" { + metadataURL = "http://169.254.169.254/latest/meta-data/" + } + + // assume 2 seconds is enough time for inside AWS network + client := &http.Client{ + Timeout: 2 * time.Second, + } + + keys := make(map[string]string) + keys["ip-address"] = "public-hostname" + keys["internal-ip"] = "local-ipv4" + + for name, key := range keys { + res, err := client.Get(metadataURL + key) + if err != nil { + // if it's a URL error, assume we're not in an AWS environment + if _, ok := err.(*url.Error); ok { + return false, nil + } + // not sure what other errors it would return + return false, err + } + body, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + return false, err + } + + // assume we want blank entries + node.Attributes["network."+name] = strings.Trim(string(body), "\n") + } + + if throughput := f.LinkSpeed(""); throughput != "" { + node.Attributes["network.throughput"] = throughput + } + + return true, nil +} + +func (f *AWSNetworkFingerprint) Interfaces() []string { + // NO OP for now + return nil +} + +func (f *AWSNetworkFingerprint) LinkSpeed(device string) string { + // This table is an approximation of network speeds based on + // http://serverfault.com/questions/324883/aws-bandwidth-and-content-delivery/326797#326797 + // which itself cites these sources: + // - http://blog.rightscale.com/2007/10/28/network-performance-within-amazon-ec2-and-to-amazon-s3/ + // - http://www.soc.napier.ac.uk/~bill/chris_p.pdf + // + // This data is meant for a loose approximation + net := make(map[string]string) + net["m4.large"] = "10MB/s" + net["m3.medium"] = "10MB/s" + net["m3.large"] = "10MB/s" + net["c4.large"] = "10MB/s" + net["c3.large"] = "10MB/s" + net["c3.xlarge"] = "10MB/s" + net["r3.large"] = "10MB/s" + net["r3.xlarge"] = "10MB/s" + net["i2.xlarge"] = "10MB/s" + net["d2.xlarge"] = "10MB/s" + net["t2.micro"] = "2MB/s" + net["t2.small"] = "2MB/s" + net["t2.medium"] = "2MB/s" + net["t2.large"] = "2MB/s" + net["m4.xlarge"] = "95MB/s" + net["m4.2xlarge"] = "95MB/s" + net["m4.4xlarge"] = "95MB/s" + net["m3.xlarge"] = "95MB/s" + net["m3.2xlarge"] = "95MB/s" + net["c4.xlarge"] = "95MB/s" + net["c4.2xlarge"] = "95MB/s" + net["c4.4xlarge"] = "95MB/s" + net["c3.2xlarge"] = "95MB/s" + net["c3.4xlarge"] = "95MB/s" + net["g2.2xlarge"] = "95MB/s" + net["r3.2xlarge"] = "95MB/s" + net["r3.4xlarge"] = "95MB/s" + net["i2.2xlarge"] = "95MB/s" + net["i2.4xlarge"] = "95MB/s" + net["d2.2xlarge"] = "95MB/s" + net["d2.4xlarge"] = "95MB/s" + net["m4.10xlarge"] = "10Gbp/s" + net["c4.8xlarge"] = "10Gbp/s" + net["c3.8xlarge"] = "10Gbp/s" + net["g2.8xlarge"] = "10Gbp/s" + net["r3.8xlarge"] = "10Gbp/s" + net["i2.8xlarge"] = "10Gbp/s" + net["d2.8xlarge"] = "10Gbp/s" + + // Query the API for the instance type, and use the table above to approximate + // the network speed + metadataURL := os.Getenv("AWS_ENV_URL") + if metadataURL == "" { + metadataURL = "http://169.254.169.254/latest/meta-data/" + } + + // assume 2 seconds is enough time for inside AWS network + client := &http.Client{ + Timeout: 2 * time.Second, + } + + res, err := client.Get(metadataURL + "instance-type") + body, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + return "" + } + + key := strings.Trim(string(body), "\n") + if v, ok := net[key]; ok { + return v + } + + return "" +} diff --git a/client/fingerprint/network_test.go b/client/fingerprint/network_test.go new file mode 100644 index 000000000..85981998f --- /dev/null +++ b/client/fingerprint/network_test.go @@ -0,0 +1,76 @@ +package fingerprint + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "runtime" + "testing" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestNetworkFingerprint_basic(t *testing.T) { + f := NetworkDefault(testLogger()) + node := &structs.Node{ + Attributes: make(map[string]string), + } + + ok, err := f.Fingerprint(&config.Config{}, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("should apply") + } + if _, ok := f.(*UnixNetworkFingerprint); !ok { + t.Fatalf("Expected a Unix type Network Fingerprinter") + } + + // Darwin uses en0 for the default device, and does not have a standard + // location for the linkspeed file, so we skip these + if "darwin" != runtime.GOOS { + assertNodeAttributeContains(t, node, "network.throughput") + assertNodeAttributeContains(t, node, "network.ip-address") + } +} + +func TestNetworkFingerprint_AWS(t *testing.T) { + // configure mock server with fixture routes, data + // TODO: Refator with the AWS ENV test + routes := routes{} + if err := json.Unmarshal([]byte(aws_routes), &routes); err != nil { + t.Fatalf("Failed to unmarshal JSON in AWS ENV test: %s", err) + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, e := range routes.Endpoints { + if r.RequestURI == e.Uri { + w.Header().Set("Content-Type", e.ContentType) + fmt.Fprintln(w, e.Body) + } + } + })) + + defer ts.Close() + os.Setenv("AWS_ENV_URL", ts.URL+"/latest/meta-data/") + + f := NetworkDefault(testLogger()) + node := &structs.Node{ + Attributes: make(map[string]string), + } + + ok, err := f.Fingerprint(&config.Config{}, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("should apply") + } + + assertNodeAttributeContains(t, node, "network.throughput") + assertNodeAttributeContains(t, node, "network.ip-address") + assertNodeAttributeContains(t, node, "network.internal-ip") +} diff --git a/client/fingerprint/network_unix.go b/client/fingerprint/network_unix.go new file mode 100644 index 000000000..280e2b6e4 --- /dev/null +++ b/client/fingerprint/network_unix.go @@ -0,0 +1,153 @@ +// +build linux darwin +package fingerprint + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" +) + +// UnixNetworkFingerprint is used to fingerprint the Network capabilities of a node +type UnixNetworkFingerprint struct { + logger *log.Logger +} + +// NewNetworkFingerprint is used to create a CPU fingerprint +func NewNetworkFingerprinter(logger *log.Logger) NetworkFingerPrinter { + f := &UnixNetworkFingerprint{logger: logger} + return f +} + +func (f *UnixNetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { + if ip := ifConfig("eth0"); ip != "" { + node.Attributes["network.ip-address"] = ip + } + + if s := f.LinkSpeed("eth0"); s != "" { + node.Attributes["network.throughput"] = s + } + + // return true, because we have a network connection + return true, nil +} + +func (f *UnixNetworkFingerprint) Interfaces() []string { + // No OP for now + return nil +} + +// LinkSpeed attempts to determine link speed, first by checking if any tools +// exist that can return the speed (ethtool for now). If no tools are found, +// fall back to /sys/class/net speed file, if it exists. +// +// The return value is in the format of "MB/s" +// +// LinkSpeed returns an empty string if no tools or sys file are found +func (f *UnixNetworkFingerprint) LinkSpeed(device string) string { + // Use LookPath to find the ethtool in the systems $PATH + // If it's not found or otherwise errors, LookPath returns and empty string + // and an error we can ignore for our purposes + ethtoolPath, _ := exec.LookPath("ethtool") + if ethtoolPath != "" { + speed := linkSpeedEthtool(ethtoolPath, device) + if speed != "" { + return speed + } + } + fmt.Println("[WARN] Ethtool not found, checking /sys/net speed file") + + // Fall back on checking a system file for link speed. + return linkSpeedSys(device) +} + +// linkSpeedSys parses the information stored in the sys diretory for the +// default device. This method retuns an empty string if the file is not found +// or cannot be read +func linkSpeedSys(device string) string { + path := fmt.Sprintf("/sys/class/net/%s/speed", device) + _, err := os.Stat(path) + if err != nil { + log.Printf("[WARN] Error getting information about net speed") + return "" + } + + // Read contents of the device/speed file + content, err := ioutil.ReadFile(path) + if err == nil { + lines := strings.Split(string(content), "\n") + // convert to MB/s + mbs, err := strconv.Atoi(lines[0]) + if err != nil { + log.Println("[WARN] Unable to parse ethtool output") + return "" + } + mbs = mbs / 8 + + return fmt.Sprintf("%dMB/s", mbs) + } + return "" +} + +// linkSpeedEthtool uses the ethtool installed on the node to gather link speed +// information. It executes the command on the device specified and parses +// out the speed. The expected format is Mbps and converted to MB/s +// Returns an empty string there is an error in parsing or executing ethtool +func linkSpeedEthtool(path, device string) string { + outBytes, err := exec.Command(path, device).Output() + if err == nil { + output := strings.TrimSpace(string(outBytes)) + re := regexp.MustCompile("Speed: [0-9]+[a-zA-Z]+/s") + m := re.FindString(output) + if m == "" { + // no matches found, output may be in a different format + log.Println("[WARN] Ethtool output did not match regex") + return "" + } + + // Split and trim the Mb/s unit from the string output + args := strings.Split(m, ": ") + raw := strings.TrimSuffix(args[1], "Mb/s") + + // convert to MB/s + mbs, err := strconv.Atoi(raw) + if err != nil { + log.Println("[WARN] Unable to parse ethtool output") + return "" + } + mbs = mbs / 8 + + return fmt.Sprintf("%dMB/s", mbs) + } + log.Printf("error calling ethtool (%s): %s", path, err) + return "" +} + +// ifConfig returns the IP Address for this node according to ifConfig, for the +// specified device. +func ifConfig(device string) string { + ifConfigPath, _ := exec.LookPath("ifconfig") + if ifConfigPath != "" { + outBytes, err := exec.Command(ifConfigPath, device).Output() + if err == nil { + output := strings.TrimSpace(string(outBytes)) + re := regexp.MustCompile("inet addr:[0-9].+") + m := re.FindString(output) + args := strings.Split(m, "inet addr:") + + return args[1] + } + log.Printf("[Err] Error calling ifconfig (%s): %s", ifConfigPath, err) + return "" + } + + log.Println("[WARN] Ethtool not found") + return "" +}