Merge pull request #356 from hashicorp/network-fingerprint-refactor

Network fingerprint refactor
2026-01-09 03:45:41 +03:00 · 2015-10-29 16:59:54 -07:00
parent f9497a5b73 5b7a7f00e5
commit 0f787a331d
2 changed files with 354 additions and 119 deletions
--- a/client/fingerprint/network_test.go
+++ b/client/fingerprint/network_test.go
@@ -1,6 +1,7 @@
 package fingerprint

 import (
+	"fmt"
 	"net"
 	"testing"

@@ -8,8 +9,136 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs"
 )

+var (
+	lo = net.Interface{
+		Index:        2,
+		MTU:          65536,
+		Name:         "lo",
+		HardwareAddr: []byte{23, 43, 54, 54},
+		Flags:        net.FlagUp | net.FlagLoopback,
+	}
+
+	eth0 = net.Interface{
+		Index:        3,
+		MTU:          1500,
+		Name:         "eth0",
+		HardwareAddr: []byte{23, 44, 54, 67},
+		Flags:        net.FlagUp | net.FlagMulticast | net.FlagBroadcast,
+	}
+
+	eth1 = net.Interface{
+		Index:        4,
+		MTU:          1500,
+		Name:         "eth1",
+		HardwareAddr: []byte{23, 44, 54, 69},
+		Flags:        net.FlagMulticast | net.FlagBroadcast,
+	}
+
+	eth2 = net.Interface{
+		Index:        4,
+		MTU:          1500,
+		Name:         "eth2",
+		HardwareAddr: []byte{23, 44, 54, 70},
+		Flags:        net.FlagUp | net.FlagBroadcast | net.FlagMulticast,
+	}
+)
+
+// A fake network detector which returns no devices
+type NetworkIntefaceDetectorNoDevices struct {
+}
+
+func (f *NetworkIntefaceDetectorNoDevices) Interfaces() ([]net.Interface, error) {
+	return make([]net.Interface, 0), nil
+}
+
+func (f *NetworkIntefaceDetectorNoDevices) InterfaceByName(name string) (*net.Interface, error) {
+	return nil, fmt.Errorf("Device with name %s doesn't exist", name)
+}
+
+func (f *NetworkIntefaceDetectorNoDevices) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	return nil, fmt.Errorf("No interfaces found for device %v", intf.Name)
+}
+
+// A fake network detector which returns only loopback
+type NetworkInterfaceDetectorOnlyLo struct {
+}
+
+func (n *NetworkInterfaceDetectorOnlyLo) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{lo}, nil
+}
+
+func (n *NetworkInterfaceDetectorOnlyLo) InterfaceByName(name string) (*net.Interface, error) {
+	if name == "lo" {
+		return &lo, nil
+	}
+
+	return nil, fmt.Errorf("No device with name %v found", name)
+}
+
+func (n *NetworkInterfaceDetectorOnlyLo) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	if intf.Name == "lo" {
+		_, ipnet1, _ := net.ParseCIDR("127.0.0.1/8")
+		_, ipnet2, _ := net.ParseCIDR("2001:DB8::/48")
+		return []net.Addr{ipnet1, ipnet2}, nil
+	}
+
+	return nil, fmt.Errorf("Can't find addresses for device: %v", intf.Name)
+}
+
+// A fake network detector which simulates the presence of multiple interfaces
+type NetworkInterfaceDetectorMultipleInterfaces struct {
+}
+
+func (n *NetworkInterfaceDetectorMultipleInterfaces) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{lo, eth0, eth1, eth2}, nil
+}
+
+func (n *NetworkInterfaceDetectorMultipleInterfaces) InterfaceByName(name string) (*net.Interface, error) {
+	var intf *net.Interface
+	switch name {
+	case "lo":
+		intf = &lo
+	case "eth0":
+		intf = &eth0
+	case "eth1":
+		intf = &eth1
+	case "eth2":
+		intf = &eth2
+	}
+	if intf != nil {
+		return intf, nil
+	}
+
+	return nil, fmt.Errorf("No device with name %v found", name)
+}
+
+func (n *NetworkInterfaceDetectorMultipleInterfaces) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	if intf.Name == "lo" {
+		_, ipnet1, _ := net.ParseCIDR("127.0.0.1/8")
+		_, ipnet2, _ := net.ParseCIDR("2001:DB8::/48")
+		return []net.Addr{ipnet1, ipnet2}, nil
+	}
+
+	if intf.Name == "eth0" {
+		_, ipnet1, _ := net.ParseCIDR("100.64.0.11/10")
+		_, ipnet2, _ := net.ParseCIDR("2005:DB6::/48")
+		return []net.Addr{ipnet1, ipnet2}, nil
+	}
+
+	if intf.Name == "eth1" {
+		_, ipnet1, _ := net.ParseCIDR("100.64.0.10/10")
+		_, ipnet2, _ := net.ParseCIDR("2003:DB8::/48")
+		return []net.Addr{ipnet1, ipnet2}, nil
+	}
+
+	if intf.Name == "eth2" {
+		return []net.Addr{}, nil
+	}
+	return nil, fmt.Errorf("Can't find addresses for device: %v", intf.Name)
+}
+
 func TestNetworkFingerprint_basic(t *testing.T) {
-	f := NewNetworkFingerprinter(testLogger())
+	f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &DefaultNetworkInterfaceDetector{}}
 	node := &structs.Node{
 		Attributes: make(map[string]string),
 	}
@@ -50,3 +179,123 @@ func TestNetworkFingerprint_basic(t *testing.T) {
 		t.Fatal("Expected Network Resource to have a non-zero bandwith")
 	}
 }
+
+func TestNetworkFingerprint_no_devices(t *testing.T) {
+	f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkIntefaceDetectorNoDevices{}}
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+	cfg := &config.Config{NetworkSpeed: 100}
+
+	ok, err := f.Fingerprint(cfg, node)
+	if err == nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if ok {
+		t.Fatalf("ok: %v", ok)
+	}
+}
+
+func TestNetworkFingerprint_default_device_absent(t *testing.T) {
+	f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkInterfaceDetectorOnlyLo{}}
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+	cfg := &config.Config{NetworkSpeed: 100, NetworkInterface: "eth0"}
+
+	ok, err := f.Fingerprint(cfg, node)
+	if err == nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if ok {
+		t.Fatalf("ok: %v", ok)
+	}
+}
+
+func TestNetworkFingerPrint_default_device(t *testing.T) {
+	f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkInterfaceDetectorOnlyLo{}}
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+	cfg := &config.Config{NetworkSpeed: 100, NetworkInterface: "lo"}
+
+	ok, err := f.Fingerprint(cfg, node)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !ok {
+		t.Fatalf("should apply")
+	}
+
+	assertNodeAttributeContains(t, node, "network.ip-address")
+
+	ip := node.Attributes["network.ip-address"]
+	match := net.ParseIP(ip)
+	if match == nil {
+		t.Fatalf("Bad IP match: %s", ip)
+	}
+
+	if node.Resources == nil || len(node.Resources.Networks) == 0 {
+		t.Fatal("Expected to find Network Resources")
+	}
+
+	// Test at least the first Network Resource
+	net := node.Resources.Networks[0]
+	if net.IP == "" {
+		t.Fatal("Expected Network Resource to not be empty")
+	}
+	if net.CIDR == "" {
+		t.Fatal("Expected Network Resource to have a CIDR")
+	}
+	if net.Device == "" {
+		t.Fatal("Expected Network Resource to have a Device Name")
+	}
+	if net.MBits == 0 {
+		t.Fatal("Expected Network Resource to have a non-zero bandwith")
+	}
+}
+
+func TestNetworkFingerPrint_excludelo_down_interfaces(t *testing.T) {
+	f := &NetworkFingerprint{logger: testLogger(), interfaceDetector: &NetworkInterfaceDetectorMultipleInterfaces{}}
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+	cfg := &config.Config{NetworkSpeed: 100}
+
+	ok, err := f.Fingerprint(cfg, node)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !ok {
+		t.Fatalf("should apply")
+	}
+
+	assertNodeAttributeContains(t, node, "network.ip-address")
+
+	ip := node.Attributes["network.ip-address"]
+	match := net.ParseIP(ip)
+	if match == nil {
+		t.Fatalf("Bad IP match: %s", ip)
+	}
+
+	if node.Resources == nil || len(node.Resources.Networks) == 0 {
+		t.Fatal("Expected to find Network Resources")
+	}
+
+	// Test at least the first Network Resource
+	net := node.Resources.Networks[0]
+	if net.IP == "" {
+		t.Fatal("Expected Network Resource to have an IP")
+	}
+	if net.CIDR == "" {
+		t.Fatal("Expected Network Resource to have a CIDR")
+	}
+	if net.Device != "eth0" {
+		t.Fatal("Expected Network Resource to be eth0. Actual: ", net.Device)
+	}
+	if net.MBits == 0 {
+		t.Fatal("Expected Network Resource to have a non-zero bandwith")
+	}
+}
--- a/client/fingerprint/network_unix.go
+++ b/client/fingerprint/network_unix.go
@@ -10,7 +10,6 @@ import (
 	"net"
 	"os/exec"
 	"regexp"
-	"runtime"
 	"strconv"
 	"strings"

@@ -20,54 +19,64 @@ import (

 // NetworkFingerprint is used to fingerprint the Network capabilities of a node
 type NetworkFingerprint struct {
-	logger *log.Logger
+	logger            *log.Logger
+	interfaceDetector NetworkInterfaceDetector
+}
+
+// An interface to isolate calls to various api in net package
+// This facilitates testing where we can implement
+// fake interfaces and addresses to test varios code paths
+type NetworkInterfaceDetector interface {
+	Interfaces() ([]net.Interface, error)
+	InterfaceByName(name string) (*net.Interface, error)
+	Addrs(intf *net.Interface) ([]net.Addr, error)
+}
+
+// Implements the interface detector which calls net directly
+type DefaultNetworkInterfaceDetector struct {
+}
+
+func (b *DefaultNetworkInterfaceDetector) Interfaces() ([]net.Interface, error) {
+	return net.Interfaces()
+}
+
+func (b *DefaultNetworkInterfaceDetector) InterfaceByName(name string) (*net.Interface, error) {
+	return net.InterfaceByName(name)
+}
+
+func (b *DefaultNetworkInterfaceDetector) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	return intf.Addrs()
 }

 // NewNetworkFingerprinter returns a new NetworkFingerprinter with the given
 // logger
 func NewNetworkFingerprinter(logger *log.Logger) Fingerprint {
-	f := &NetworkFingerprint{logger: logger}
+	f := &NetworkFingerprint{logger: logger, interfaceDetector: &DefaultNetworkInterfaceDetector{}}
 	return f
 }

 func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
 	// newNetwork is populated and addded to the Nodes resources
 	newNetwork := &structs.NetworkResource{}
-	defaultDevice := ""
-	ip := ""
+	var ip string

-	// 1. Use user-defined network device
-	// 2. Use first interface found in the system for non-dev mode. (dev mode uses lo by default.)
-	if cfg.NetworkInterface != "" {
-		defaultDevice = cfg.NetworkInterface
-		ip = f.ipAddress(defaultDevice)
-	} else {
-
-		intfs, err := net.Interfaces()
-		if err != nil {
-			return false, err
-		}
-
-		for _, i := range intfs {
-			if (i.Flags&net.FlagUp != 0) && (i.Flags&(net.FlagLoopback|net.FlagPointToPoint) == 0) {
-				if ip = f.ipAddress(i.Name); ip != "" {
-					defaultDevice = i.Name
-					break
-				}
-			}
-		}
+	intf, err := f.findInterface(cfg.NetworkInterface)
+	if err != nil {
+		return false, fmt.Errorf("Error while detecting network interface during fingerprinting: %v", err)
 	}

-	if (defaultDevice != "") && (ip != "") {
-		newNetwork.Device = defaultDevice
-		node.Attributes["network.ip-address"] = ip
-		newNetwork.IP = ip
-		newNetwork.CIDR = newNetwork.IP + "/32"
-	} else {
-		return false, fmt.Errorf("Unable to find any network interface which has IP address")
+	if ip, err = f.ipAddress(intf); err != nil {
+		return false, fmt.Errorf("Unable to find IP address of interface: %s, err: %v", intf.Name, err)
 	}

-	if throughput := f.linkSpeed(defaultDevice); throughput > 0 {
+	newNetwork.Device = intf.Name
+	node.Attributes["network.ip-address"] = ip
+	newNetwork.IP = ip
+	newNetwork.CIDR = newNetwork.IP + "/32"
+
+	f.logger.Printf("[DEBUG] fingerprint.network: Detected interface %v  with IP %v during fingerprinting", intf.Name, ip)
+
+	if throughput := f.linkSpeed(intf.Name); throughput > 0 {
 		newNetwork.MBits = throughput
 	} else {
 		f.logger.Printf("[DEBUG] fingerprint.network: Unable to read link speed; setting to default %v", cfg.NetworkSpeed)
@@ -152,99 +161,76 @@ func (f *NetworkFingerprint) linkSpeedEthtool(path, device string) int {
 	return mbs
 }

-// ipAddress returns the first IPv4 address on the configured default interface
-// Tries Golang native functions and falls back onto ifconfig
-func (f *NetworkFingerprint) ipAddress(device string) string {
-	if ip, err := f.nativeIpAddress(device); err == nil {
-		return ip
+// Gets the ipv4 addr for a network interface
+func (f *NetworkFingerprint) ipAddress(intf *net.Interface) (string, error) {
+	var addrs []net.Addr
+	var err error
+
+	if addrs, err = f.interfaceDetector.Addrs(intf); err != nil {
+		return "", err
 	}

-	return f.ifConfig(device)
-}
-
-func (f *NetworkFingerprint) nativeIpAddress(device string) (string, error) {
-	// Find IP address on configured interface
-	var ip string
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return "", errors.New("could not retrieve interface list")
+	if len(addrs) == 0 {
+		return "", errors.New(fmt.Sprintf("Interface %s has no IP address", intf.Name))
 	}
-
-	// TODO: should we handle IPv6 here? How do we determine precedence?
-	for _, i := range ifaces {
-		if i.Name != device {
-			continue
+	for _, addr := range addrs {
+		var ip net.IP
+		switch v := (addr).(type) {
+		case *net.IPNet:
+			ip = v.IP
+		case *net.IPAddr:
+			ip = v.IP
 		}
-
-		addrs, err := i.Addrs()
-		if err != nil {
-			return "", errors.New("could not retrieve interface IP addresses")
-		}
-
-		for _, a := range addrs {
-			switch v := a.(type) {
-			case *net.IPNet:
-				if v.IP.To4() != nil {
-					ip = v.IP.String()
-				}
-			case *net.IPAddr:
-				if v.IP.To4() != nil {
-					ip = v.IP.String()
-				}
-			}
+		if ip.To4() != nil {
+			return ip.String(), nil
 		}
 	}

-	if net.ParseIP(ip) == nil {
-		return "", errors.New(fmt.Sprintf("could not parse IP address `%s`", ip))
-	}
+	return "", fmt.Errorf("Couldn't parse IP address for interface %s", intf.Name)

-	return ip, nil
 }

-// ifConfig returns the IP Address for this node according to ifConfig, for the
-// specified device.
-func (f *NetworkFingerprint) ifConfig(device string) string {
-	ifConfigPath, _ := exec.LookPath("ifconfig")
-	if ifConfigPath == "" {
-		f.logger.Println("[WARN] fingerprint.network: ifconfig not found")
-		return ""
-	}
-
-	outBytes, err := exec.Command(ifConfigPath, device).Output()
-	if err != nil {
-		f.logger.Printf("[WARN] fingerprint.network: Error calling ifconfig (%s %s): %v", ifConfigPath, device, err)
-		return ""
-	}
-
-	// Parse out the IP address returned from ifconfig for this device
-	// Tested on Ubuntu, the matching part of ifconfig output for eth0 is like
-	// so:
-	//   inet addr:10.0.2.15  Bcast:10.0.2.255  Mask:255.255.255.0
-	// For OS X and en0, we have:
-	//  inet 192.168.0.7 netmask 0xffffff00 broadcast 192.168.0.255
-	output := strings.TrimSpace(string(outBytes))
-
-	// re is a regular expression, which can vary based on the OS
-	var re *regexp.Regexp
-
-	if "darwin" == runtime.GOOS {
-		re = regexp.MustCompile("inet [0-9].+")
-	} else {
-		re = regexp.MustCompile("inet addr:[0-9].+")
-	}
-	args := strings.Split(re.FindString(output), " ")
-
-	var ip string
-	if len(args) > 1 {
-		ip = strings.TrimPrefix(args[1], "addr:")
-	}
-
-	// validate what we've sliced out is a valid IP
-	if net.ParseIP(ip) == nil {
-		f.logger.Printf("[WARN] fingerprint.network: Unable to parse IP in output of '%s %s'", ifConfigPath, device)
-		return ""
-	}
-
-	return ip
+// Checks if the device is marked UP by the operator
+func (f *NetworkFingerprint) isDeviceEnabled(intf *net.Interface) bool {
+	return intf.Flags&net.FlagUp != 0
+}
+
+// Checks if the device has any IP address configured
+func (f *NetworkFingerprint) deviceHasIpAddress(intf *net.Interface) bool {
+	_, err := f.ipAddress(intf)
+	return err == nil
+}
+
+func (n *NetworkFingerprint) isDeviceLoopBackOrPointToPoint(intf *net.Interface) bool {
+	return intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) != 0
+}
+
+// Returns the interface with the name passed by user
+// If the name is blank then it iterates through all the devices
+// and finds one which is routable and marked as UP
+// It excludes PPP and lo devices unless they are specifically asked
+func (f *NetworkFingerprint) findInterface(deviceName string) (*net.Interface, error) {
+	var interfaces []net.Interface
+	var err error
+
+	if deviceName != "" {
+		return f.interfaceDetector.InterfaceByName(deviceName)
+	}
+
+	var intfs []net.Interface
+
+	if intfs, err = f.interfaceDetector.Interfaces(); err != nil {
+		return nil, err
+	}
+
+	for _, intf := range intfs {
+		if f.isDeviceEnabled(&intf) && !f.isDeviceLoopBackOrPointToPoint(&intf) && f.deviceHasIpAddress(&intf) {
+			interfaces = append(interfaces, intf)
+		}
+	}
+
+	if len(interfaces) == 0 {
+		return nil, errors.New("No network interfaces were detected")
+	}
+	return &interfaces[0], nil
 }