Generate bootMAC and change boot network to NAT

Change-Id: Id8776e8153c57682ed02f4e6534d8dfda4ed25a1

config/crd/bases/airship.airshipit.org_vinoes.yaml

@@ -77,10 +77,6 @@ spec:
                   vinoBuilderImage:
                     type: string
                 type: object
-              managementPhysicalInterfaceName:
-                description: ManagementPhysicalInterfaceName will be used to connect
-                  to libvirt network
-                type: string
               networks:
                 description: Define network parameters
                 items:
@@ -151,11 +147,6 @@ spec:
                         that will be created These labels will override keys from
                         k8s node, that are specified in vino.NodeLabelKeysToCopy
                       type: object
-                    bootInterfaceName:
-                      description: BootInterfaceName references the interface name
-                        in the list of NetworkInterfaces Vino will take this interface
-                        find its mac address and use it as bootMACAddress for BMH
-                      type: string
                     count:
                       type: integer
                     diskDrives:
@@ -216,6 +207,10 @@ spec:
                             type: string
                         type: object
                       type: array
+                    rootDeviceName:
+                      description: RootDeviceName is the root device for underlying
+                        VM, /dev/vda for example default is /dev/vda
+                      type: string
                   type: object
                 type: array
               pxeBootImageHost:

config/manager/flavor-templates.yaml

@@ -70,6 +70,7 @@ flavorTemplates:
 
           <interface type='network'>
             <source network='pxe'/>
+            <mac address='{{ domain.bootMACAddress }}'/>
             <model type='virtio'/>
           </interface>
 

config/manager/network-templates.yaml

@@ -3,7 +3,7 @@ libvirtNetworks:
     libvirtTemplate: |
       <network>
         <name>pxe</name>
-        <forward mode='route'/>
+        <forward mode='nat'/>
         <bridge name='pxe' stp='off' delay='0'/>
         <ip address='10.153.241.1' netmask='255.255.255.0'>
           <!-- <tftp root='/srv/tftp'/> -->

config/samples/vino_cr.yaml

@@ -16,7 +16,7 @@ spec:
   networks:
     - name: vm-infra
       subnet: 192.168.2.0/20
-      type: ipv4
+      type: bridge
       allocationStart: 192.168.2.10
       allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
       routes:

config/samples/vino_cr_4_workers_1_cp.yaml

@@ -33,7 +33,6 @@ spec:
       networkDataTemplate:
         name: "test-template"
         namespace: "default"
-      bootInterfaceName: pxe
       networkInterfaces:
         - name: management
           type: network
@@ -46,7 +45,6 @@ spec:
       networkDataTemplate:
         name: "test-template"
         namespace: "default"
-      bootInterfaceName: pxe
       networkInterfaces:
         - name: management
           type: network

docs/api/vino.md

@@ -127,16 +127,6 @@ string
 </tr>
 <tr>
 <td>
-<code>managementPhysicalInterfaceName</code><br>
-<em>
-string
-</em>
-</td>
-<td>
-</td>
-</tr>
-<tr>
-<td>
 <code>pxeBootImageHost</code><br>
 <em>
 string
@@ -247,6 +237,16 @@ string
 </tr>
 <tr>
 <td>
+<code>bootMACAddress</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+</td>
+</tr>
+<tr>
+<td>
 <code>interfaces</code><br>
 <em>
 <a href="#airship.airshipit.org/v1.BuilderNetworkInterface">
@@ -1087,14 +1087,14 @@ NamespacedName
 </tr>
 <tr>
 <td>
-<code>bootInterfaceName</code><br>
+<code>rootDeviceName</code><br>
 <em>
 string
 </em>
 </td>
 <td>
-<p>BootInterfaceName references the interface name in the list of NetworkInterfaces
+<p>RootDeviceName is the root device for underlying VM, /dev/vda for example
-Vino will take this interface find its mac address and use it as bootMACAddress for BMH</p>
+default is /dev/vda</p>
 </td>
 </tr>
 </tbody>
@@ -1337,17 +1337,6 @@ and place them on BMHs that correspond to this node</p>
 </tr>
 <tr>
 <td>
-<code>managementPhysicalInterfaceName</code><br>
-<em>
-string
-</em>
-</td>
-<td>
-<p>ManagementPhysicalInterfaceName will be used to connect to libvirt network</p>
-</td>
-</tr>
-<tr>
-<td>
 <code>pxeBootImageHost</code><br>
 <em>
 string
@@ -1508,17 +1497,6 @@ and place them on BMHs that correspond to this node</p>
 </tr>
 <tr>
 <td>
-<code>managementPhysicalInterfaceName</code><br>
-<em>
-string
-</em>
-</td>
-<td>
-<p>ManagementPhysicalInterfaceName will be used to connect to libvirt network</p>
-</td>
-</tr>
-<tr>
-<td>
 <code>pxeBootImageHost</code><br>
 <em>
 string

pkg/api/v1/vino_builder.go

@@ -19,7 +19,6 @@ package v1
 // TODO (kkalynovskyi) create an API object for this, and refactor vino-builder to read it from kubernetes.
 type Builder struct {
 	GWIPBridge           string `json:"gwIPBridge,omitempty"`
-	ManagementPhysicalInterfaceName string `json:"managementPhysicalInterfaceName,omitempty"`
 	PXEBootImageHost     string `json:"pxeBootImageHost,omitempty"`
 	PXEBootImageHostPort int    `json:"pxeBootImageHostPort,omitempty"`
 
@@ -33,13 +32,14 @@ type Builder struct {
 type BuilderNetworkInterface struct {
 	IPAddress        string `json:"ipAddress,omitempty"`
 	MACAddress       string `json:"macAddress,omitempty"`
-	NetworkInterface
+	NetworkInterface `json:",inline"`
 }
 
 // BuilderDomain represents a VINO libvirt domain
 type BuilderDomain struct {
 	Name           string `json:"name,omitempty"`
 	Role           string `json:"role,omitempty"`
+	BootMACAddress string `json:"bootMACAddress,omitempty"`
 
 	Interfaces []BuilderNetworkInterface `json:"interfaces,omitempty"`
 }

pkg/api/v1/vino_types.go

@@ -39,6 +39,8 @@ const (
 	VinoNodeNetworkValuesAnnotation = "airshipit.org/vino.network-values"
 	// VinoNetworkDataTemplateDefaultKey expected template key networkdata template secret for vino node
 	VinoNetworkDataTemplateDefaultKey = "template"
+	// VinoDefaultRootDeviceName is default root device for the underlying libvirt VM
+	VinoDefaultRootDeviceName = "/dev/sda"
 )
 
 // Constants for BasicAuth
@@ -69,8 +71,6 @@ type VinoSpec struct {
 	// NodeLabelKeysToCopy vino controller will get these labels from k8s nodes
 	// and place them on BMHs that correspond to this node
 	NodeLabelKeysToCopy []string `json:"nodeLabelKeysToCopy,omitempty"`
-	// ManagementPhysicalInterfaceName will be used to connect to libvirt network
-	ManagementPhysicalInterfaceName string `json:"managementPhysicalInterfaceName,omitempty"`
 	// PXEBootImageHost will be used to download the PXE boot image
 	PXEBootImageHost string `json:"pxeBootImageHost,omitempty"`
 	// PXEBootImageHostPort will be used to download the PXE boot image
@@ -135,9 +135,9 @@ type NodeSet struct {
 	DiskDrives                []DiskDrivesTemplate `json:"diskDrives,omitempty"`
 	// NetworkDataTemplate must have a template key
 	NetworkDataTemplate NamespacedName `json:"networkDataTemplate,omitempty"`
-	// BootInterfaceName references the interface name in the list of NetworkInterfaces
+	// RootDeviceName is the root device for underlying VM, /dev/vda for example
-	// Vino will take this interface find its mac address and use it as bootMACAddress for BMH
+	// default is /dev/vda
-	BootInterfaceName string `json:"bootInterfaceName,omitempty"`
+	RootDeviceName string `json:"rootDeviceName,omitempty"`
 }
 
 // NamespacedName to be used to spawn VMs

pkg/managers/bmh.go

@@ -43,7 +43,6 @@ const (
 
 type networkTemplateValues struct {
 	BMHName string
-	BootMACAddress string
 
 	Node     vinov1.NodeSet // the specific node type to be templated
 	Networks []vinov1.Network
@@ -55,6 +54,7 @@ type BMHManager struct {
 
 	client.Client
 	ViNO        *vinov1.Vino
+	BootNetwork *vinov1.Network
 	Ipam        *ipam.Ipam
 	Logger      logr.Logger
 
@@ -160,7 +160,32 @@ func (r *BMHManager) requestVMs(ctx context.Context) error {
 }
 
 func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino) error {
-	for _, network := range vino.Spec.Networks {
+	// TODO (kkalynovskyi) these needs to be propagated into network template, and be configurable
+	// TODO (kkalynovskyi) develop generic network templates that would allow to handle all networks
+	// in single generic way.
+	// Bootnetwork needs to be handled spearately because it needs to be created by libvirt
+	// And have different configuration.
+	if r.BootNetwork == nil {
+		r.BootNetwork = &vinov1.Network{
+			SubNet:          "10.153.241.0/24",
+			AllocationStart: "10.153.241.2",
+			AllocationStop:  "10.153.241.254",
+			Name:            "pxe-boot",
+			MACPrefix: "52:54:00:32:00:00",
+		}
+	}
+	networks := vino.Spec.Networks
+	// Append bootnetwork to be created in IPAM
+	networks = append(networks, *r.BootNetwork)
+	for _, network := range networks {
+		if err := r.createIpamNetwork(ctx, network); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *BMHManager) createIpamNetwork(ctx context.Context, network vinov1.Network) error {
 	subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
 	if err != nil {
 		return err
@@ -171,12 +196,7 @@ func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino)
 			"default prefix", DefaultMACPrefix, "network name", network.Name)
 		macPrefix = DefaultMACPrefix
 	}
-		err = r.Ipam.AddSubnetRange(ctx, network.SubNet, subnetRange, macPrefix)
+	return r.Ipam.AddSubnetRange(ctx, network.SubNet, subnetRange, macPrefix)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
 }
 
 func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
@@ -223,6 +243,11 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
 				labels[label] = value
 			}
 
+			rootDeviceName := node.RootDeviceName
+			if rootDeviceName == "" {
+				rootDeviceName = vinov1.VinoDefaultRootDeviceName
+			}
+
 			credentialSecretName := r.setBMHCredentials(bmhName)
 			bmh := &metal3.BareMetalHost{
 				ObjectMeta: metav1.ObjectMeta{
@@ -241,6 +266,9 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
 						DisableCertificateVerification: true,
 					},
 					BootMACAddress: domainValues.BootMACAddress,
+					RootDeviceHints: &metal3.RootDeviceHints{
+						DeviceName: rootDeviceName,
+					},
 				},
 			}
 			r.bmhList = append(r.bmhList, bmh)
@@ -251,7 +279,6 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
 	vinoBuilder := vinov1.Builder{
 		PXEBootImageHost:     r.ViNO.Spec.PXEBootImageHost,
 		PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort,
-		ManagementPhysicalInterfaceName: r.ViNO.Spec.ManagementPhysicalInterfaceName,
 		Networks:             r.ViNO.Spec.Networks,
 		Nodes:                r.ViNO.Spec.Nodes,
 		CPUConfiguration:     r.ViNO.Spec.CPUConfiguration,
@@ -287,8 +314,6 @@ func (r *BMHManager) domainSpecificNetValues(
 	// Allocate an IP for each of this BMH's network interfaces
 
 	domainInterfaces := []vinov1.BuilderNetworkInterface{}
-
-	var bootMAC string
 	for _, iface := range node.NetworkInterfaces {
 		networkName := iface.NetworkName
 		subnet := ""
@@ -297,8 +322,7 @@ func (r *BMHManager) domainSpecificNetValues(
 		for _, network := range networks {
 			if network.Name == networkName {
 				subnet = network.SubNet
-				subnetRange, err = ipam.NewRange(network.AllocationStart,
+				subnetRange, err = ipam.NewRange(network.AllocationStart, network.AllocationStop)
-					network.AllocationStop)
 				if err != nil {
 					return networkTemplateValues{}, err
 				}
@@ -318,23 +342,38 @@ func (r *BMHManager) domainSpecificNetValues(
 			MACAddress:       macAddress,
 			NetworkInterface: iface,
 		})
-		if iface.Name == node.BootInterfaceName {
+
-			bootMAC = macAddress
-		}
 		r.Logger.Info("Got MAC and IP for the network and node",
-			"MAC", macAddress, "IP", ipAddress, "bmh name", bmhName, "bootMAC", bootMAC)
+			"MAC", macAddress, "IP", ipAddress, "bmh name", bmhName)
 	}
+	// Handle bootMAC separately
+	bootMAC, err := r.generatePXEBootMAC(ctx, bmhName)
+	if err != nil {
+		return networkTemplateValues{}, err
+	}
+	r.Logger.Info("Got bootMAC address for BMH node", "bmh name", bmhName, "bootMAC", bootMAC)
 	return networkTemplateValues{
 		Node:     node,
 		BMHName:  bmhName,
 		Networks: networks,
-		BootMACAddress: bootMAC,
 		BuilderDomain: vinov1.BuilderDomain{
+			BootMACAddress: bootMAC,
 			Interfaces:     domainInterfaces,
 		},
 	}, nil
 }
 
+func (r *BMHManager) generatePXEBootMAC(ctx context.Context, bmhName string) (string, error) {
+	subnetRange, err := ipam.NewRange(r.BootNetwork.AllocationStart, r.BootNetwork.AllocationStop)
+	if err != nil {
+		return "", err
+	}
+
+	ipAllocatedTo := fmt.Sprintf("%s/%s", bmhName, "pxe-boot")
+	_, mac, err := r.Ipam.AllocateIP(ctx, r.BootNetwork.SubNet, subnetRange, ipAllocatedTo)
+	return mac, err
+}
+
 func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vinoBuilder vinov1.Builder) error {
 	b, err := yaml.Marshal(vinoBuilder)
 	if err != nil {