diff --git a/client/fingerprint/network_test.go b/client/fingerprint/network_test.go index 077d5a573..a42b1d7b3 100644 --- a/client/fingerprint/network_test.go +++ b/client/fingerprint/network_test.go @@ -1,6 +1,7 @@ package fingerprint import ( + "fmt" "net" "testing" @@ -8,8 +9,136 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) +var ( + lo = net.Interface{ + Index: 2, + MTU: 65536, + Name: "lo", + HardwareAddr: []byte{23, 43, 54, 54}, + Flags: net.FlagUp | net.FlagLoopback, + } + + eth0 = net.Interface{ + Index: 3, + MTU: 1500, + Name: "eth0", + HardwareAddr: []byte{23, 44, 54, 67}, + Flags: net.FlagUp | net.FlagMulticast | net.FlagBroadcast, + } + + eth1 = net.Interface{ + Index: 4, + MTU: 1500, + Name: "eth1", + HardwareAddr: []byte{23, 44, 54, 69}, + Flags: net.FlagMulticast | net.FlagBroadcast, + } + + eth2 = net.Interface{ + Index: 4, + MTU: 1500, + Name: "eth2", + HardwareAddr: []byte{23, 44, 54, 70}, + Flags: net.FlagUp | net.FlagBroadcast | net.FlagMulticast, + } +) + +// A fake network detector which returns no devices +type NetworkIntefaceDetectorNoDevices struct { +} + +func (f *NetworkIntefaceDetectorNoDevices) Interfaces() ([]net.Interface, error) { + return make([]net.Interface, 0), nil +} + +func (f *NetworkIntefaceDetectorNoDevices) InterfaceByName(name string) (*net.Interface, error) { + return nil, fmt.Errorf("Device with name %s doesn't exist", name) +} + +func (f *NetworkIntefaceDetectorNoDevices) Addrs(intf *net.Interface) ([]net.Addr, error) { + return nil, fmt.Errorf("No interfaces found for device %v", intf.Name) +} + +// A fake network detector which returns only loopback +type NetworkInterfaceDetectorOnlyLo struct { +} + +func (n *NetworkInterfaceDetectorOnlyLo) Interfaces() ([]net.Interface, error) { + return []net.Interface{lo}, nil +} + +func (n *NetworkInterfaceDetectorOnlyLo) InterfaceByName(name string) (*net.Interface, error) { + if name == "lo" { + return &lo, nil + } + + return nil, fmt.Errorf("No device with name %v found", name) +} + +func (n *NetworkInterfaceDetectorOnlyLo) Addrs(intf *net.Interface) ([]net.Addr, error) { + if intf.Name == "lo" { + _, ipnet1, _ := net.ParseCIDR("127.0.0.1/8") + _, ipnet2, _ := net.ParseCIDR("2001:DB8::/48") + return []net.Addr{ipnet1, ipnet2}, nil + } + + return nil, fmt.Errorf("Can't find addresses for device: %v", intf.Name) +} + +// A fake network detector which simulates the presence of multiple interfaces +type NetworkInterfaceDetectorMultipleInterfaces struct { +} + +func (n *NetworkInterfaceDetectorMultipleInterfaces) Interfaces() ([]net.Interface, error) { + return []net.Interface{lo, eth0, eth1, eth2}, nil +} + +func (n *NetworkInterfaceDetectorMultipleInterfaces) InterfaceByName(name string) (*net.Interface, error) { + var intf *net.Interface + switch name { + case "lo": + intf = &lo + case "eth0": + intf = ð0 + case "eth1": + intf = ð1 + case "eth2": + intf = ð2 + } + if intf != nil { + return intf, nil + } + + return nil, fmt.Errorf("No device with name %v found", name) +} + +func (n *NetworkInterfaceDetectorMultipleInterfaces) Addrs(intf *net.Interface) ([]net.Addr, error) { + if intf.Name == "lo" { + _, ipnet1, _ := net.ParseCIDR("127.0.0.1/8") + _, ipnet2, _ := net.ParseCIDR("2001:DB8::/48") + return []net.Addr{ipnet1, ipnet2}, nil + } + + if intf.Name == "eth0" { + _, ipnet1, _ := net.ParseCIDR("100.64.0.11/10") + _, ipnet2, _ := net.ParseCIDR("2005:DB6::/48") + return []net.Addr{ipnet1, ipnet2}, nil + } + + if intf.Name == "eth1" { + _, ipnet1, _ := net.ParseCIDR("100.64.0.10/10") + _, ipnet2, _ := net.ParseCIDR("2003:DB8::/48") + return []net.Addr{ipnet1, ipnet2}, nil + } + + if intf.Name == "eth2" { + return []net.Addr{}, nil + } + return nil, fmt.Errorf("Can't find addresses for device: %v", intf.Name) +} + func TestNetworkFingerprint_basic(t *testing.T) { - f := NewNetworkFingerprinter(testLogger()) + f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &DefaultNetworkInterfaceDetector{}} node := &structs.Node{ Attributes: make(map[string]string), } @@ -50,3 +179,123 @@ func TestNetworkFingerprint_basic(t *testing.T) { t.Fatal("Expected Network Resource to have a non-zero bandwith") } } + +func TestNetworkFingerprint_no_devices(t *testing.T) { + f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkIntefaceDetectorNoDevices{}} + node := &structs.Node{ + Attributes: make(map[string]string), + } + cfg := &config.Config{NetworkSpeed: 100} + + ok, err := f.Fingerprint(cfg, node) + if err == nil { + t.Fatalf("err: %v", err) + } + + if ok { + t.Fatalf("ok: %v", ok) + } +} + +func TestNetworkFingerprint_default_device_absent(t *testing.T) { + f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkInterfaceDetectorOnlyLo{}} + node := &structs.Node{ + Attributes: make(map[string]string), + } + cfg := &config.Config{NetworkSpeed: 100, NetworkInterface: "eth0"} + + ok, err := f.Fingerprint(cfg, node) + if err == nil { + t.Fatalf("err: %v", err) + } + + if ok { + t.Fatalf("ok: %v", ok) + } +} + +func TestNetworkFingerPrint_default_device(t *testing.T) { + f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkInterfaceDetectorOnlyLo{}} + node := &structs.Node{ + Attributes: make(map[string]string), + } + cfg := &config.Config{NetworkSpeed: 100, NetworkInterface: "lo"} + + ok, err := f.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("should apply") + } + + assertNodeAttributeContains(t, node, "network.ip-address") + + ip := node.Attributes["network.ip-address"] + match := net.ParseIP(ip) + if match == nil { + t.Fatalf("Bad IP match: %s", ip) + } + + if node.Resources == nil || len(node.Resources.Networks) == 0 { + t.Fatal("Expected to find Network Resources") + } + + // Test at least the first Network Resource + net := node.Resources.Networks[0] + if net.IP == "" { + t.Fatal("Expected Network Resource to not be empty") + } + if net.CIDR == "" { + t.Fatal("Expected Network Resource to have a CIDR") + } + if net.Device == "" { + t.Fatal("Expected Network Resource to have a Device Name") + } + if net.MBits == 0 { + t.Fatal("Expected Network Resource to have a non-zero bandwith") + } +} + +func TestNetworkFingerPrint_excludelo_down_interfaces(t *testing.T) { + f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkInterfaceDetectorMultipleInterfaces{}} + node := &structs.Node{ + Attributes: make(map[string]string), + } + cfg := &config.Config{NetworkSpeed: 100} + + ok, err := f.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("should apply") + } + + assertNodeAttributeContains(t, node, "network.ip-address") + + ip := node.Attributes["network.ip-address"] + match := net.ParseIP(ip) + if match == nil { + t.Fatalf("Bad IP match: %s", ip) + } + + if node.Resources == nil || len(node.Resources.Networks) == 0 { + t.Fatal("Expected to find Network Resources") + } + + // Test at least the first Network Resource + net := node.Resources.Networks[0] + if net.IP == "" { + t.Fatal("Expected Network Resource to have an IP") + } + if net.CIDR == "" { + t.Fatal("Expected Network Resource to have a CIDR") + } + if net.Device != "eth0" { + t.Fatal("Expected Network Resource to be eth0. Actual: ", net.Device) + } + if net.MBits == 0 { + t.Fatal("Expected Network Resource to have a non-zero bandwith") + } +} diff --git a/client/fingerprint/network_unix.go b/client/fingerprint/network_unix.go index 9a422ec01..4278384e9 100644 --- a/client/fingerprint/network_unix.go +++ b/client/fingerprint/network_unix.go @@ -10,7 +10,6 @@ import ( "net" "os/exec" "regexp" - "runtime" "strconv" "strings" @@ -20,54 +19,64 @@ import ( // NetworkFingerprint is used to fingerprint the Network capabilities of a node type NetworkFingerprint struct { - logger *log.Logger + logger *log.Logger + interfaceDetector NetworkInterfaceDetector +} + +// An interface to isolate calls to various api in net package +// This facilitates testing where we can implement +// fake interfaces and addresses to test varios code paths +type NetworkInterfaceDetector interface { + Interfaces() ([]net.Interface, error) + InterfaceByName(name string) (*net.Interface, error) + Addrs(intf *net.Interface) ([]net.Addr, error) +} + +// Implements the interface detector which calls net directly +type DefaultNetworkInterfaceDetector struct { +} + +func (b *DefaultNetworkInterfaceDetector) Interfaces() ([]net.Interface, error) { + return net.Interfaces() +} + +func (b *DefaultNetworkInterfaceDetector) InterfaceByName(name string) (*net.Interface, error) { + return net.InterfaceByName(name) +} + +func (b *DefaultNetworkInterfaceDetector) Addrs(intf *net.Interface) ([]net.Addr, error) { + return intf.Addrs() } // NewNetworkFingerprinter returns a new NetworkFingerprinter with the given // logger func NewNetworkFingerprinter(logger *log.Logger) Fingerprint { - f := &NetworkFingerprint{logger: logger} + f := &NetworkFingerprint{logger: logger, interfaceDetector: &DefaultNetworkInterfaceDetector{}} return f } func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { // newNetwork is populated and addded to the Nodes resources newNetwork := &structs.NetworkResource{} - defaultDevice := "" - ip := "" + var ip string - // 1. Use user-defined network device - // 2. Use first interface found in the system for non-dev mode. (dev mode uses lo by default.) - if cfg.NetworkInterface != "" { - defaultDevice = cfg.NetworkInterface - ip = f.ipAddress(defaultDevice) - } else { - - intfs, err := net.Interfaces() - if err != nil { - return false, err - } - - for _, i := range intfs { - if (i.Flags&net.FlagUp != 0) && (i.Flags&(net.FlagLoopback|net.FlagPointToPoint) == 0) { - if ip = f.ipAddress(i.Name); ip != "" { - defaultDevice = i.Name - break - } - } - } + intf, err := f.findInterface(cfg.NetworkInterface) + if err != nil { + return false, fmt.Errorf("Error while detecting network interface during fingerprinting: %v", err) } - if (defaultDevice != "") && (ip != "") { - newNetwork.Device = defaultDevice - node.Attributes["network.ip-address"] = ip - newNetwork.IP = ip - newNetwork.CIDR = newNetwork.IP + "/32" - } else { - return false, fmt.Errorf("Unable to find any network interface which has IP address") + if ip, err = f.ipAddress(intf); err != nil { + return false, fmt.Errorf("Unable to find IP address of interface: %s, err: %v", intf.Name, err) } - if throughput := f.linkSpeed(defaultDevice); throughput > 0 { + newNetwork.Device = intf.Name + node.Attributes["network.ip-address"] = ip + newNetwork.IP = ip + newNetwork.CIDR = newNetwork.IP + "/32" + + f.logger.Printf("[DEBUG] fingerprint.network: Detected interface %v with IP %v during fingerprinting", intf.Name, ip) + + if throughput := f.linkSpeed(intf.Name); throughput > 0 { newNetwork.MBits = throughput } else { f.logger.Printf("[DEBUG] fingerprint.network: Unable to read link speed; setting to default %v", cfg.NetworkSpeed) @@ -152,99 +161,76 @@ func (f *NetworkFingerprint) linkSpeedEthtool(path, device string) int { return mbs } -// ipAddress returns the first IPv4 address on the configured default interface -// Tries Golang native functions and falls back onto ifconfig -func (f *NetworkFingerprint) ipAddress(device string) string { - if ip, err := f.nativeIpAddress(device); err == nil { - return ip +// Gets the ipv4 addr for a network interface +func (f *NetworkFingerprint) ipAddress(intf *net.Interface) (string, error) { + var addrs []net.Addr + var err error + + if addrs, err = f.interfaceDetector.Addrs(intf); err != nil { + return "", err } - return f.ifConfig(device) -} - -func (f *NetworkFingerprint) nativeIpAddress(device string) (string, error) { - // Find IP address on configured interface - var ip string - ifaces, err := net.Interfaces() - if err != nil { - return "", errors.New("could not retrieve interface list") + if len(addrs) == 0 { + return "", errors.New(fmt.Sprintf("Interface %s has no IP address", intf.Name)) } - - // TODO: should we handle IPv6 here? How do we determine precedence? - for _, i := range ifaces { - if i.Name != device { - continue + for _, addr := range addrs { + var ip net.IP + switch v := (addr).(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP } - - addrs, err := i.Addrs() - if err != nil { - return "", errors.New("could not retrieve interface IP addresses") - } - - for _, a := range addrs { - switch v := a.(type) { - case *net.IPNet: - if v.IP.To4() != nil { - ip = v.IP.String() - } - case *net.IPAddr: - if v.IP.To4() != nil { - ip = v.IP.String() - } - } + if ip.To4() != nil { + return ip.String(), nil } } - if net.ParseIP(ip) == nil { - return "", errors.New(fmt.Sprintf("could not parse IP address `%s`", ip)) - } + return "", fmt.Errorf("Couldn't parse IP address for interface %s", intf.Name) - return ip, nil } -// ifConfig returns the IP Address for this node according to ifConfig, for the -// specified device. -func (f *NetworkFingerprint) ifConfig(device string) string { - ifConfigPath, _ := exec.LookPath("ifconfig") - if ifConfigPath == "" { - f.logger.Println("[WARN] fingerprint.network: ifconfig not found") - return "" - } - - outBytes, err := exec.Command(ifConfigPath, device).Output() - if err != nil { - f.logger.Printf("[WARN] fingerprint.network: Error calling ifconfig (%s %s): %v", ifConfigPath, device, err) - return "" - } - - // Parse out the IP address returned from ifconfig for this device - // Tested on Ubuntu, the matching part of ifconfig output for eth0 is like - // so: - // inet addr:10.0.2.15 Bcast:10.0.2.255 Mask:255.255.255.0 - // For OS X and en0, we have: - // inet 192.168.0.7 netmask 0xffffff00 broadcast 192.168.0.255 - output := strings.TrimSpace(string(outBytes)) - - // re is a regular expression, which can vary based on the OS - var re *regexp.Regexp - - if "darwin" == runtime.GOOS { - re = regexp.MustCompile("inet [0-9].+") - } else { - re = regexp.MustCompile("inet addr:[0-9].+") - } - args := strings.Split(re.FindString(output), " ") - - var ip string - if len(args) > 1 { - ip = strings.TrimPrefix(args[1], "addr:") - } - - // validate what we've sliced out is a valid IP - if net.ParseIP(ip) == nil { - f.logger.Printf("[WARN] fingerprint.network: Unable to parse IP in output of '%s %s'", ifConfigPath, device) - return "" - } - - return ip +// Checks if the device is marked UP by the operator +func (f *NetworkFingerprint) isDeviceEnabled(intf *net.Interface) bool { + return intf.Flags&net.FlagUp != 0 +} + +// Checks if the device has any IP address configured +func (f *NetworkFingerprint) deviceHasIpAddress(intf *net.Interface) bool { + _, err := f.ipAddress(intf) + return err == nil +} + +func (n *NetworkFingerprint) isDeviceLoopBackOrPointToPoint(intf *net.Interface) bool { + return intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) != 0 +} + +// Returns the interface with the name passed by user +// If the name is blank then it iterates through all the devices +// and finds one which is routable and marked as UP +// It excludes PPP and lo devices unless they are specifically asked +func (f *NetworkFingerprint) findInterface(deviceName string) (*net.Interface, error) { + var interfaces []net.Interface + var err error + + if deviceName != "" { + return f.interfaceDetector.InterfaceByName(deviceName) + } + + var intfs []net.Interface + + if intfs, err = f.interfaceDetector.Interfaces(); err != nil { + return nil, err + } + + for _, intf := range intfs { + if f.isDeviceEnabled(&intf) && !f.isDeviceLoopBackOrPointToPoint(&intf) && f.deviceHasIpAddress(&intf) { + interfaces = append(interfaces, intf) + } + } + + if len(interfaces) == 0 { + return nil, errors.New("No network interfaces were detected") + } + return &interfaces[0], nil }