diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index f004110b4..cc787248f 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -44,6 +44,11 @@ func (n *Node) Register(args *structs.NodeRegisterRequest, reply *structs.NodeUp return fmt.Errorf("invalid status for node") } + // Compute the node class + if err := args.Node.ComputeClass(); err != nil { + return fmt.Errorf("failed to computed node class: %v", err) + } + // Commit this update via Raft _, index, err := n.srv.raftApply(structs.NodeRegisterRequestType, args) if err != nil { diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index efa6880ba..e6fc889f8 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -45,6 +45,9 @@ func TestClientEndpoint_Register(t *testing.T) { if out.CreateIndex != resp.Index { t.Fatalf("index mis-match") } + if out.ComputedClass == 0 { + t.Fatal("ComputedClass not set") + } } func TestClientEndpoint_Deregister(t *testing.T) { diff --git a/nomad/structs/node_class.go b/nomad/structs/node_class.go new file mode 100644 index 000000000..7cae65ef7 --- /dev/null +++ b/nomad/structs/node_class.go @@ -0,0 +1,82 @@ +package structs + +import ( + "fmt" + "strings" + + "github.com/mitchellh/hashstructure" +) + +const ( + // A suffix that can be appended to node meta keys to mark them for + // exclusion in computed node class. + NodeMetaUnique = "_unique" +) + +// ComputeClass computes a derived class for the node based on its attributes. +// ComputedClass is a unique id that identifies nodes with a common set of +// attributes and capabilities. Thus, when calculating a node's computed class +// we avoid including any uniquely identifing fields. +func (n *Node) ComputeClass() error { + // TODO: Bucket node resources such as DiskMB/IOPS/etc. + hash, err := hashstructure.Hash(n, nil) + if err != nil { + return err + } + + n.ComputedClass = hash + return nil +} + +// HashInclude is used to blacklist uniquely identifying node fields from being +// included in the computed node class. +func (n Node) HashInclude(field string, v interface{}) (bool, error) { + switch field { + case "ID", "Name", "Links": // Uniquely identifying + return false, nil + case "Drain", "Status", "StatusDescription": // Set by server + return false, nil + case "ComputedClass", "UniqueAttributes": // Part of computed node class + return false, nil + case "CreateIndex", "ModifyIndex": // Raft indexes + return false, nil + case "Reserved": // Doesn't effect placement capability + return false, nil + default: + return true, nil + } +} + +// HashIncludeMap is used to blacklist uniquely identifying node map keys from being +// included in the computed node class. +func (n Node) HashIncludeMap(field string, k, v interface{}) (bool, error) { + key, ok := k.(string) + if !ok { + return false, fmt.Errorf("map key %v not a string") + } + + switch field { + case "Attributes": + // Check if the key is marked as unique by the fingerprinters. + _, unique := n.UniqueAttributes[key] + return !unique, nil + case "Meta": + // Check if the user marked the key as unique. + return !strings.HasSuffix(key, NodeMetaUnique), nil + default: + return false, fmt.Errorf("unexpected map field: %v", field) + } +} + +// HashInclude is used to blacklist uniquely identifying network fields from being +// included in the computed node class. +func (n NetworkResource) HashInclude(field string, v interface{}) (bool, error) { + switch field { + case "IP", "CIDR": // Uniquely identifying + return false, nil + case "ReservedPorts", "DynamicPorts": // Doesn't effect placement capability + return false, nil + default: + return true, nil + } +} diff --git a/nomad/structs/node_class_test.go b/nomad/structs/node_class_test.go new file mode 100644 index 000000000..dae377d94 --- /dev/null +++ b/nomad/structs/node_class_test.go @@ -0,0 +1,229 @@ +package structs + +import ( + "testing" +) + +func testNode() *Node { + return &Node{ + ID: GenerateUUID(), + Datacenter: "dc1", + Name: "foobar", + Attributes: map[string]string{ + "kernel.name": "linux", + "arch": "x86", + "version": "0.1.0", + "driver.exec": "1", + }, + UniqueAttributes: make(map[string]struct{}), + Resources: &Resources{ + CPU: 4000, + MemoryMB: 8192, + DiskMB: 100 * 1024, + IOPS: 150, + Networks: []*NetworkResource{ + &NetworkResource{ + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + }, + Reserved: &Resources{ + CPU: 100, + MemoryMB: 256, + DiskMB: 4 * 1024, + Networks: []*NetworkResource{ + &NetworkResource{ + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []Port{{Label: "main", Value: 22}}, + MBits: 1, + }, + }, + }, + Links: map[string]string{ + "consul": "foobar.dc1", + }, + Meta: map[string]string{ + "pci-dss": "true", + }, + NodeClass: "linux-medium-pci", + Status: NodeStatusReady, + } +} + +func TestNode_ComputedClass(t *testing.T) { + // Create a node and gets it computed class + n := testNode() + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + old := n.ComputedClass + + // Compute again to ensure determinism + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if old != n.ComputedClass { + t.Fatalf("ComputeClass() should have returned same class; got %v; want %v", n.ComputedClass, old) + } + + // Modify a field and compute the class again. + n.Datacenter = "New DC" + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + + if old == n.ComputedClass { + t.Fatal("ComputeClass() returned same computed class") + } +} + +func TestNode_ComputedClass_Ignore(t *testing.T) { + // Create a node and gets it computed class + n := testNode() + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + old := n.ComputedClass + + // Modify an ignored field and compute the class again. + n.ID = "New ID" + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + + if old != n.ComputedClass { + t.Fatal("ComputeClass() should have ignored field") + } +} + +func TestNode_ComputedClass_NetworkResources(t *testing.T) { + // Create a node with a few network resources and gets it computed class + nr1 := &NetworkResource{ + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + } + nr2 := &NetworkResource{ + Device: "eth1", + CIDR: "192.168.0.100/32", + MBits: 500, + } + n := &Node{ + Resources: &Resources{ + Networks: []*NetworkResource{nr1, nr2}, + }, + } + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + old := n.ComputedClass + + // Change the order of the network resources and compute the class again. + n.Resources.Networks = []*NetworkResource{nr2, nr1} + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + + if old != n.ComputedClass { + t.Fatal("ComputeClass() didn't ignore NetworkResource order") + } + +} + +func TestNode_ComputedClass_Attr(t *testing.T) { + // Create a node and gets it computed class + n := testNode() + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + old := n.ComputedClass + + // Modify an attribute and compute the class again. + n.Attributes["version"] = "New Version" + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + if old == n.ComputedClass { + t.Fatal("ComputeClass() ignored attribute change") + } + old = n.ComputedClass + + // Add an ignored attribute and compute the class again. + key := "ignore" + n.Attributes[key] = "hello world" + n.UniqueAttributes[key] = struct{}{} + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + if old != n.ComputedClass { + t.Fatal("ComputeClass() didn't ignore unique attribute") + } +} + +func TestNode_ComputedClass_Meta(t *testing.T) { + // Create a node and gets it computed class + n := testNode() + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + old := n.ComputedClass + + // Modify a meta key and compute the class again. + n.Meta["pci-dss"] = "false" + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + if old == n.ComputedClass { + t.Fatal("ComputeClass() ignored meta change") + } + old = n.ComputedClass + + // Add a unique meta key and compute the class again. + key := "test_unique" + n.Meta[key] = "ignore" + if err := n.ComputeClass(); err != nil { + t.Fatalf("ComputeClass() failed: %v", err) + } + if n.ComputedClass == 0 { + t.Fatal("ComputeClass() didn't set computed class") + } + if old != n.ComputedClass { + t.Fatal("ComputeClass() didn't ignore unique meta key") + } +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 810126953..de6bfc096 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -476,6 +476,9 @@ type Node struct { // "docker.runtime=1.8.3" Attributes map[string]string + // UniqueAttributes are attributes that uniquely identify a node. + UniqueAttributes map[string]struct{} + // Resources is the available resources on the client. // For example 'cpu=2' 'memory=2048' Resources *Resources @@ -500,6 +503,10 @@ type Node struct { // together for the purpose of determining scheduling pressure. NodeClass string + // ComputedClass is a unique id that identifies nodes with a common set of + // attributes and capabilities. + ComputedClass uint64 + // Drain is controlled by the servers, and not the client. // If true, no jobs will be scheduled to this node, and existing // allocations will be drained. @@ -563,7 +570,7 @@ type Resources struct { MemoryMB int `mapstructure:"memory"` DiskMB int `mapstructure:"disk"` IOPS int - Networks []*NetworkResource + Networks []*NetworkResource `hash:"set"` } // Copy returns a deep copy of the resources