Merge branch 'master' into f-port-configs

This commit is contained in:
Chris Bednarski
2015-09-23 15:31:00 -07:00
11 changed files with 592 additions and 12 deletions

View File

@@ -386,7 +386,7 @@ func (c *Client) setupNode() error {
// fingerprint is used to fingerprint the client and setup the node
func (c *Client) fingerprint() error {
var applied []string
for name := range fingerprint.BuiltinFingerprints {
for _, name := range fingerprint.BuiltinFingerprints {
f, err := fingerprint.NewFingerprint(name, c.logger)
if err != nil {
return err

View File

@@ -77,7 +77,6 @@ func (d *JavaDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool,
// OpenJDK Runtime Environment (IcedTea6 1.13.8) (6b36-1.13.8-0ubuntu1~12.04)
// OpenJDK 64-Bit Server VM (build 23.25-b01, mixed mode)
// Each line is terminated by \n
info := strings.Split(infoString, "\n")
versionString := info[0]
versionString = strings.TrimPrefix(versionString, "java version ")

View File

@@ -53,11 +53,10 @@ func TestQemuDriver_Start(t *testing.T) {
task := &structs.Task{
Config: map[string]string{
"image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img",
"checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544",
"accelerator": "tcg",
"host_port": "8080",
"guest_port": "8081",
"checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544",
// ssh u/p would be here
},
}

View File

@@ -6,6 +6,7 @@ import (
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
@@ -13,6 +14,54 @@ import (
"github.com/hashicorp/nomad/nomad/structs"
)
// map of instance type to approximate speed, in Mbits/s
// http://serverfault.com/questions/324883/aws-bandwidth-and-content-delivery/326797#326797
// which itself cites these sources:
// - http://blog.rightscale.com/2007/10/28/network-performance-within-amazon-ec2-and-to-amazon-s3/
// - http://www.soc.napier.ac.uk/~bill/chris_p.pdf
//
// This data is meant for a loose approximation
var ec2InstanceSpeedMap = map[string]int{
"m4.large": 80,
"m3.medium": 80,
"m3.large": 80,
"c4.large": 80,
"c3.large": 80,
"c3.xlarge": 80,
"r3.large": 80,
"r3.xlarge": 80,
"i2.xlarge": 80,
"d2.xlarge": 80,
"t2.micro": 16,
"t2.small": 16,
"t2.medium": 16,
"t2.large": 16,
"m4.xlarge": 760,
"m4.2xlarge": 760,
"m4.4xlarge": 760,
"m3.xlarge": 760,
"m3.2xlarge": 760,
"c4.xlarge": 760,
"c4.2xlarge": 760,
"c4.4xlarge": 760,
"c3.2xlarge": 760,
"c3.4xlarge": 760,
"g2.2xlarge": 760,
"r3.2xlarge": 760,
"r3.4xlarge": 760,
"i2.2xlarge": 760,
"i2.4xlarge": 760,
"d2.2xlarge": 760,
"d2.4xlarge": 760,
"m4.10xlarge": 10000,
"c4.8xlarge": 10000,
"c3.8xlarge": 10000,
"g2.8xlarge": 10000,
"r3.8xlarge": 10000,
"i2.8xlarge": 10000,
"d2.8xlarge": 10000,
}
// EnvAWSFingerprint is used to fingerprint the CPU
type EnvAWSFingerprint struct {
logger *log.Logger
@@ -25,6 +74,15 @@ func NewEnvAWSFingerprint(logger *log.Logger) Fingerprint {
}
func (f *EnvAWSFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
if !isAWS() {
return false, nil
}
// newNetwork is populated and addded to the Nodes resources
newNetwork := &structs.NetworkResource{
Device: "eth0",
}
if node.Links == nil {
node.Links = make(map[string]string)
}
@@ -63,7 +121,7 @@ func (f *EnvAWSFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
resp, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
log.Fatal(err)
f.logger.Printf("[ERR]: fingerprint.env_aws: Error reading response body for AWS %s", k)
}
// assume we want blank entries
@@ -71,8 +129,95 @@ func (f *EnvAWSFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
node.Attributes["platform.aws."+key] = strings.Trim(string(resp), "\n")
}
// populate links
// copy over network specific information
if node.Attributes["platform.aws.local-ipv4"] != "" {
node.Attributes["network.ip-address"] = node.Attributes["platform.aws.local-ipv4"]
newNetwork.IP = node.Attributes["platform.aws.local-ipv4"]
newNetwork.CIDR = newNetwork.IP + "/32"
}
// find LinkSpeed from lookup
if throughput := f.linkSpeed(); throughput > 0 {
newNetwork.MBits = throughput
}
if node.Resources == nil {
node.Resources = &structs.Resources{}
}
node.Resources.Networks = append(node.Resources.Networks, newNetwork)
// populate Node Network Resources
// populate Links
node.Links["aws.ec2"] = node.Attributes["platform.aws.placement.availability-zone"] + "." + node.Attributes["platform.aws.instance-id"]
return true, nil
}
func isAWS() bool {
// Read the internal metadata URL from the environment, allowing test files to
// provide their own
metadataURL := os.Getenv("AWS_ENV_URL")
if metadataURL == "" {
metadataURL = "http://169.254.169.254/latest/meta-data/"
}
// assume 2 seconds is enough time for inside AWS network
client := &http.Client{
Timeout: 2 * time.Second,
}
// Query the metadata url for the ami-id, to veryify we're on AWS
resp, err := client.Get(metadataURL + "ami-id")
if err != nil {
log.Printf("[ERR] fingerprint.env_aws: Error querying AWS Metadata URL, skipping")
return false
}
defer resp.Body.Close()
instanceID, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("[ERR] fingerprint.env_aws: Error reading AWS Instance ID, skipping")
return false
}
match, err := regexp.MatchString("ami-*", string(instanceID))
if !match {
return false
}
return true
}
// EnvAWSFingerprint uses lookup table to approximate network speeds
func (f *EnvAWSFingerprint) linkSpeed() int {
// Query the API for the instance type, and use the table above to approximate
// the network speed
metadataURL := os.Getenv("AWS_ENV_URL")
if metadataURL == "" {
metadataURL = "http://169.254.169.254/latest/meta-data/"
}
// assume 2 seconds is enough time for inside AWS network
client := &http.Client{
Timeout: 2 * time.Second,
}
res, err := client.Get(metadataURL + "instance-type")
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
f.logger.Printf("[ERR]: fingerprint.env_aws: Error reading response body for instance-type")
return 0
}
key := strings.Trim(string(body), "\n")
v, ok := ec2InstanceSpeedMap[key]
if !ok {
return 0
}
return v
}

View File

@@ -69,6 +69,8 @@ func TestEnvAWSFingerprint_aws(t *testing.T) {
"platform.aws.public-hostname",
"platform.aws.public-ipv4",
"platform.aws.placement.availability-zone",
"network.ip-address",
"network.internal-ip",
}
for _, k := range keys {
@@ -145,3 +147,69 @@ const aws_routes = `
]
}
`
func TestNetworkFingerprint_AWS(t *testing.T) {
// configure mock server with fixture routes, data
// TODO: Refator with the AWS ENV test
routes := routes{}
if err := json.Unmarshal([]byte(aws_routes), &routes); err != nil {
t.Fatalf("Failed to unmarshal JSON in AWS ENV test: %s", err)
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for _, e := range routes.Endpoints {
if r.RequestURI == e.Uri {
w.Header().Set("Content-Type", e.ContentType)
fmt.Fprintln(w, e.Body)
}
}
}))
defer ts.Close()
os.Setenv("AWS_ENV_URL", ts.URL+"/latest/meta-data/")
f := NewEnvAWSFingerprint(testLogger())
node := &structs.Node{
Attributes: make(map[string]string),
}
ok, err := f.Fingerprint(&config.Config{}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if !ok {
t.Fatalf("should apply")
}
assertNodeAttributeContains(t, node, "network.ip-address")
if node.Resources == nil || len(node.Resources.Networks) == 0 {
t.Fatal("Expected to find Network Resources")
}
// Test at least the first Network Resource
net := node.Resources.Networks[0]
if net.IP == "" {
t.Fatal("Expected Network Resource to have an IP")
}
if net.CIDR == "" {
t.Fatal("Expected Network Resource to have a CIDR")
}
if net.Device == "" {
t.Fatal("Expected Network Resource to have a Device Name")
}
}
func TestNetworkFingerprint_notAWS(t *testing.T) {
f := NewEnvAWSFingerprint(testLogger())
node := &structs.Node{
Attributes: make(map[string]string),
}
ok, err := f.Fingerprint(&config.Config{}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if ok {
t.Fatalf("Should not apply")
}
}

View File

@@ -8,21 +8,35 @@ import (
"github.com/hashicorp/nomad/nomad/structs"
)
// BuiltinFingerprints contains the built in registered fingerprints
// which are available
var BuiltinFingerprints = map[string]Factory{
// BuiltinFingerprints is a slice containing the key names of all regestered
// fingerprints available, to provided an ordered iteration
var BuiltinFingerprints = []string{
"arch",
"cpu",
"host",
"memory",
"storage",
"network",
"env_aws",
}
// builtinFingerprintMap contains the built in registered fingerprints
// which are available, corresponding to a key found in BuiltinFingerprints
var builtinFingerprintMap = map[string]Factory{
"arch": NewArchFingerprint,
"cpu": NewCPUFingerprint,
"host": NewHostFingerprint,
"memory": NewMemoryFingerprint,
"storage": NewStorageFingerprint,
"network": NewNetworkFingerprinter,
"env_aws": NewEnvAWSFingerprint,
}
// NewFingerprint is used to instantiate and return a new fingerprint
// given the name and a logger
func NewFingerprint(name string, logger *log.Logger) (Fingerprint, error) {
// Lookup the factory function
factory, ok := BuiltinFingerprints[name]
factory, ok := builtinFingerprintMap[name]
if !ok {
return nil, fmt.Errorf("unknown fingerprint '%s'", name)
}

View File

@@ -0,0 +1,48 @@
package fingerprint
import (
"net"
"testing"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
func TestNetworkFingerprint_basic(t *testing.T) {
f := NewNetworkFingerprinter(testLogger())
node := &structs.Node{
Attributes: make(map[string]string),
}
ok, err := f.Fingerprint(&config.Config{}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if !ok {
t.Fatalf("should apply")
}
assertNodeAttributeContains(t, node, "network.ip-address")
ip := node.Attributes["network.ip-address"]
match := net.ParseIP(ip)
if match == nil {
t.Fatalf("Bad IP match: %s", ip)
}
if node.Resources == nil || len(node.Resources.Networks) == 0 {
t.Fatal("Expected to find Network Resources")
}
// Test at least the first Network Resource
net := node.Resources.Networks[0]
if net.IP == "" {
t.Fatal("Expected Network Resource to not be empty")
}
if net.CIDR == "" {
t.Fatal("Expected Network Resource to have a CIDR")
}
if net.Device == "" {
t.Fatal("Expected Network Resource to have a Device Name")
}
}

View File

@@ -0,0 +1,192 @@
// +build linux darwin
package fingerprint
import (
"fmt"
"io/ioutil"
"log"
"net"
"os"
"os/exec"
"regexp"
"runtime"
"strconv"
"strings"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
// NetworkFingerprint is used to fingerprint the Network capabilities of a node
type NetworkFingerprint struct {
logger *log.Logger
}
// NewNetworkFingerprint returns a new NetworkFingerprinter with the given
// logger
func NewNetworkFingerprinter(logger *log.Logger) Fingerprint {
f := &NetworkFingerprint{logger: logger}
return f
}
func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// newNetwork is populated and addded to the Nodes resources
newNetwork := &structs.NetworkResource{}
// eth0 is the default device for Linux, and en0 is default for OS X
defaultDevice := "eth0"
if "darwin" == runtime.GOOS {
defaultDevice = "en0"
}
newNetwork.Device = defaultDevice
if ip := f.ifConfig(defaultDevice); ip != "" {
node.Attributes["network.ip-address"] = ip
newNetwork.IP = ip
newNetwork.CIDR = newNetwork.IP + "/32"
}
if throughput := f.linkSpeed("eth0"); throughput > 0 {
newNetwork.MBits = throughput
}
if node.Resources == nil {
node.Resources = &structs.Resources{}
}
node.Resources.Networks = append(node.Resources.Networks, newNetwork)
// return true, because we have a network connection
return true, nil
}
// LinkSpeed attempts to determine link speed, first by checking if any tools
// exist that can return the speed (ethtool for now). If no tools are found,
// fall back to /sys/class/net speed file, if it exists.
//
// The return value is in the format of "<int>MB/s"
//
// LinkSpeed returns an empty string if no tools or sys file are found
func (f *NetworkFingerprint) linkSpeed(device string) int {
// Use LookPath to find the ethtool in the systems $PATH
// If it's not found or otherwise errors, LookPath returns and empty string
// and an error we can ignore for our purposes
ethtoolPath, _ := exec.LookPath("ethtool")
if ethtoolPath != "" {
if speed := f.linkSpeedEthtool(ethtoolPath, device); speed > 0 {
return speed
}
}
f.logger.Printf("[WARN] fingerprint.network: Ethtool not found, checking /sys/net speed file")
// Fall back on checking a system file for link speed.
return f.linkSpeedSys(device)
}
// linkSpeedSys parses the information stored in the sys diretory for the
// default device. This method retuns an empty string if the file is not found
// or cannot be read
func (f *NetworkFingerprint) linkSpeedSys(device string) int {
path := fmt.Sprintf("/sys/class/net/%s/speed", device)
_, err := os.Stat(path)
if err != nil {
f.logger.Printf("[WARN] fingerprint.network: Error getting information about net speed")
return 0
}
// Read contents of the device/speed file
content, err := ioutil.ReadFile(path)
if err == nil {
lines := strings.Split(string(content), "\n")
// convert to MB/s
mbs, err := strconv.Atoi(lines[0])
if err != nil {
f.logger.Println("[WARN] fingerprint.network: Enable to parse ethtool output")
return 0
}
if mbs > 0 {
return mbs
}
}
return 0
}
// linkSpeedEthtool uses the ethtool installed on the node to gather link speed
// information. It executes the command on the device specified and parses
// out the speed. The expected format is Mbps and converted to MB/s
// Returns an empty string there is an error in parsing or executing ethtool
func (f *NetworkFingerprint) linkSpeedEthtool(path, device string) int {
outBytes, err := exec.Command(path, device).Output()
if err == nil {
output := strings.TrimSpace(string(outBytes))
re := regexp.MustCompile("Speed: [0-9]+[a-zA-Z]+/s")
m := re.FindString(output)
if m == "" {
// no matches found, output may be in a different format
f.logger.Println("[WARN] fingerprint.network: Ethtool output did not match regex")
return 0
}
// Split and trim the Mb/s unit from the string output
args := strings.Split(m, ": ")
raw := strings.TrimSuffix(args[1], "Mb/s")
// convert to MB/s
mbs, err := strconv.Atoi(raw)
if err != nil {
f.logger.Println("[WARN] fingerprint.network: Unable to parse ethtool output")
return 0
}
if mbs > 0 {
return mbs
}
}
f.logger.Printf("[ERR] fingerprint.network: Error calling ethtool (%s): %s", path, err)
return 0
}
// ifConfig returns the IP Address for this node according to ifConfig, for the
// specified device.
func (f *NetworkFingerprint) ifConfig(device string) string {
ifConfigPath, _ := exec.LookPath("ifconfig")
if ifConfigPath != "" {
outBytes, err := exec.Command(ifConfigPath, device).Output()
if err == nil {
// Parse out the IP address returned from ifconfig for this device
// Tested on Ubuntu, the matching part of ifconfig output for eth0 is like
// so:
// inet addr:10.0.2.15 Bcast:10.0.2.255 Mask:255.255.255.0
// For OS X and en0, we have:
// inet 192.168.0.7 netmask 0xffffff00 broadcast 192.168.0.255
output := strings.TrimSpace(string(outBytes))
// re is a regular expression, which can vary based on the OS
var re *regexp.Regexp
if "darwin" == runtime.GOOS {
re = regexp.MustCompile("inet [0-9].+")
} else {
re = regexp.MustCompile("inet addr:[0-9].+")
}
args := strings.Split(re.FindString(output), " ")
var ip string
if len(args) > 1 {
ip = strings.TrimPrefix(args[1], "addr:")
}
// validate what we've sliced out is a valid IP
if net.ParseIP(ip) != nil {
return ip
}
}
f.logger.Printf("[ERR] fingerprint.network: Error calling ifconfig (%s): %s", ifConfigPath, err)
return ""
}
f.logger.Println("[WARN] fingerprint.network: Ethtool not found")
return ""
}

View File

@@ -0,0 +1,26 @@
// +build windows
package fingerprint
import (
"log"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
// NetworkFingerprint is used to fingerprint the Network capabilities of a node
type NetworkFingerprint struct {
logger *log.Logger
}
// NewNetworkFingerprint returns a new NetworkFingerprinter with the given
// logger
func NewNetworkFingerprinter(logger *log.Logger) Fingerprint {
f := &NetworkFingerprint{logger: logger}
return f
}
func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// return false, because we don't yet support Windows
return false, nil
}

View File

@@ -10,5 +10,45 @@ description: |-
Name: `java`
TODO
The `Java` driver is used to execute Java applications packaged into a Java Jar
file. The driver currently requires the Jar file be accessible via
HTTP from the Nomad client.
## Task Configuration
The `java` driver supports the following configuration in the job spec:
* `jar_source` - **(Required)** The hosted location of the source Jar file. Must be accessible
from the Nomad client, via HTTP
* `args` - (Optional) The argument list for the `java` command, space separated.
## Client Requirements
The `java` driver requires Java to be installed and in your systems `$PATH`.
The `jar_source` must be accessible by the node running Nomad. This can be an
internal source, private to your cluster, but it must be reachable by the client
over HTTP.
## Client Attributes
The `java` driver will set the following client attributes:
* `driver.java` - Set to `1` if Java is found on the host node. Nomad determines
this by executing `java -version` on the host and parsing the output
* `driver.java.version` - Version of Java, ex: `1.6.0_65`
* `driver.java.runtime` - Runtime version, ex: `Java(TM) SE Runtime Environment (build 1.6.0_65-b14-466.1-11M4716)`
* `driver.java.vm` - Virtual Machine information, ex: `Java HotSpot(TM) 64-Bit Server VM (build 20.65-b04-466.1, mixed mode)`
## Resource Isolation
The resource isolation provided varies by the operating system of
the client and the configuration.
On Linux, Nomad will attempt to use cgroups, namespaces, and chroot
to isolate the resources of a process. If the Nomad agent is not
running as root many of these mechanisms cannot be used.
As a baseline, the Java jars will be ran inside a Java Virtual Machine,
providing a minimum amount of isolation.

View File

@@ -10,5 +10,54 @@ description: |-
Name: `qemu`
TODO
The `Qemu` driver provides a generic virtual machine runner. Qemu can utilize
the KVM kernel module to utilize hardware virtualization features and provide
great performance. Currently the `Qemu` driver can map a set of ports from the
host machine to the guest virtual machine, and provides configuration for
resource allocation.
The `Qemu` driver can execute any regular `qemu` image (e.g. `qcow`, `img`,
`iso`), and is currently invoked with `qemu-system-x86_64`.
## Task Configuration
The `Qemu` driver supports the following configuration in the job spec:
* `image_source` - **(Required)** The hosted location of the source Qemu image. Must be accessible
from the Nomad client, via HTTP.
* `checksum` - **(Required)** The MD5 checksum of the `qemu` image. If the
checksums do not match, the `Qemu` diver will fail to start the image
* `accelerator` - (Optional) The type of accelerator to use in the invocation.
If the host machine has `Qemu` installed with KVM support, users can specify `kvm` for the `accelerator`. Default is `tcg`
* `host_port` - **(Required)** Port on the host machine to forward to the guest
VM
* `guest_port` - **(Required)** Port on the guest machine that is listening for
traffic from the host
## Client Requirements
The `Qemu` driver requires Qemu to be installed and in your systems `$PATH`.
The `image_source` must be accessible by the node running Nomad. This can be an
internal source, private to your cluster, but it must be reachable by the client
over HTTP.
## Client Attributes
The `Qemu` driver will set the following client attributes:
* `driver.qemu` - Set to `1` if Qemu is found on the host node. Nomad determines
this by executing `qemu-system-x86_64 -version` on the host and parsing the output
* `driver.qemu.version` - Version of `qemu-system-x86_64, ex: `2.4.0`
## Resource Isolation
The resource isolation provided varies by the operating system of
the client and the configuration.
On Linux, Nomad will attempt to use cgroups, namespaces, and chroot
to isolate the resources of a process. If the Nomad agent is not
running as root many of these mechanisms cannot be used.
As a baseline, the Qemu images will be ran inside a virtual machine operated by
Qemu, providing a minimum amount of isolation.