Merge "Integrate vino-builder"

config/crd/bases/airship.airshipit.org_vinoes.yaml

@@ -150,6 +150,7 @@ spec:
                   count:
                     type: integer
                   diskDrives:
+                    items:
                       description: DiskDrivesTemplate defines disks on the VM
                       properties:
                         name:
@@ -167,6 +168,7 @@ spec:
                         type:
                           type: string
                       type: object
+                    type: array
                   libvirtTemplate:
                     description: NamespacedName to be used to spawn VMs
                     properties:
@@ -176,7 +178,7 @@ spec:
                         type: string
                     type: object
                   name:
-                    description: Parameter for Node master or worker-standard
+                    description: Parameter for Node master or worker
                     type: string
                   networkDataTemplate:
                     description: NetworkDataTemplate must have a template key

config/manager/daemonset-template.yaml

@@ -19,7 +19,7 @@ spec:
         - name: libvirt
           command:
             - /tmp/libvirt.sh
-          image: quay.io/airshipit/libvirt
+          image: quay.io/airshipit/libvirt:latest-ubuntu_bionic
           securityContext:
             privileged: true
             runAsUser: 0
@@ -59,6 +59,48 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+        - name: vino-builder
+          command:
+              - bash
+              - -cex
+              - |
+                /entrypoint.sh || true
+                tail -f /dev/null
+          securityContext:
+            privileged: true
+            runAsUser: 0
+            readOnlyRootFilesystem: false
+          ports:
+            - containerPort: 8001
+              hostPort: 8001
+          image: quay.io/airshipit/vino-builder:latest-ubuntu_bionic
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: flavors
+              mountPath: /var/lib/vino-builder/flavors
+            - name: flavor-templates
+              mountPath: /var/lib/vino-builder/flavor-templates
+            - name: network-templates
+              mountPath: /var/lib/vino-builder/network-templates
+            - name: storage-templates
+              mountPath: /var/lib/vino-builder/storage-templates
+            - name: pod-tmp
+              mountPath: /tmp
+            - mountPath: /lib/modules
+              name: libmodules
+              readOnly: true
+            - name: var-lib-libvirt
+              mountPath: /var/lib/libvirt
+            - name: var-run-libvirt
+              mountPath: /var/run/libvirt
+            - name: run
+              mountPath: /run
+            - name: dev
+              mountPath: /dev
+            - name: cgroup
+              mountPath: /sys/fs/cgroup
+            - name: logs
+              mountPath: /var/log/libvirt
       volumes:
         - name: libmodules
           hostPath:
@@ -84,3 +126,21 @@ spec:
         - name: var-run-libvirt
           hostPath:
             path: /var/run/libvirt
+        - name: flavors
+          configMap:
+            name: vino-flavors
+            defaultMode: 0555
+        - name: flavor-templates
+          configMap:
+            name: vino-flavor-templates
+            defaultMode: 0555
+        - name: network-templates
+          configMap:
+            name: vino-network-templates
+            defaultMode: 0555
+        - name: storage-templates
+          configMap:
+            name: vino-storage-templates
+            defaultMode: 0555
+        - name: pod-tmp
+          emptyDir: {}

config/manager/flavor-templates.yaml

@@ -0,0 +1,229 @@
+flavorTemplates:
+  master:
+    domainTemplate: |
+      {% set nodename = 'master-' + item|string %}
+      <domain type="kvm">
+        <name>{{ nodename }}</name>
+        <uuid>{{ nodename | hash('md5') }}</uuid>
+        <metadata>
+          <vino:flavor>master</vino:flavor>
+          <vino:creationTime>{{ ansible_date_time.date }}</vino:creationTime>
+        </metadata>
+        <memory unit="GiB">{{ flavors.master.memory }}</memory>
+        {% if flavors.worker.hugepages is defined and flavors.worker.hugepages == true %}
+        <memoryBacking>
+          <hugepages>
+            <page size='1' unit='GiB' />
+          </hugepages>
+        </memoryBacking>
+        {% endif %}
+        <vcpu placement="static">{{ flavors.master.vcpus }}</vcpu>
+        {% if node_core_map[nodename] is defined %}
+        # function to produce list of cpus, in same numa (controled by bool), state will need to be tracked via file on hypervisor host. gotpl psudo:
+        <cputune>
+          <shares>8192</shares>
+          {% for core in node_core_map[nodename] %}
+          <vcpupin vcpu="{{ loop.index0 }}" cpuset="{{ core }}"/>
+          {% endfor %}
+          <emulatorpin cpuset="{{ node_core_map[nodename]|join(',') }}"/>
+        </cputune>
+        {% endif %}
+        <resource>
+          <partition>/machine</partition>
+        </resource>
+        <os>
+          <type arch="x86_64" machine="pc">hvm</type>
+          <boot dev="hd"/>
+        </os>
+        <features>
+          <acpi/>
+          <apic/>
+        </features>
+        <cpu mode="host-passthrough" />
+        <clock offset="utc">
+          <timer name="pit" tickpolicy="delay"/>
+          <timer name="rtc" tickpolicy="catchup"/>
+          <timer name="hpet" present="no"/>
+        </clock>
+        <on_poweroff>destroy</on_poweroff>
+        <on_reboot>restart</on_reboot>
+        <on_crash>destroy</on_crash>
+        <devices>
+          <emulator>/usr/bin/qemu-system-x86_64</emulator>
+
+          # for each disk requested
+          <disk type='volume' device='disk'>
+            <driver name="qemu" type="qcow2" cache="none" discard="unmap"/>
+            <source pool='vino-default' volume='{{ nodename }}'/>
+            <target dev='vde' bus='virtio'/>
+          </disk>
+
+          <controller type="usb" index="0" model="piix3-uhci">
+            <alias name="usb"/>
+            <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x2"/>
+          </controller>
+          <controller type="pci" index="0" model="pci-root">
+            <alias name="pci.0"/>
+          </controller>
+          <controller type="ide" index="0">
+            <alias name="ide"/>
+            <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
+          </controller>
+
+          # for each interface defined in vino, e.g.
+          <interface type='bridge'>
+            <mac address='52:54:00:83:e9:f9'/>
+            <source bridge='management'/>
+            <model type='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+          </interface>
+
+          <serial type="pty">
+            <source path="/dev/pts/3"/>
+            <log file="/var/lib/vino/instances/{{ nodename }}.console.log" append="off"/>
+            <target type="isa-serial" port="0">
+              <model name="isa-serial"/>
+            </target>
+            <alias name="serial0"/>
+          </serial>
+          <console type="pty" tty="/dev/pts/3">
+            <source path="/dev/pts/3"/>
+            <log file="/var/lib/vino/instances/{{ nodename }}.console.log" append="off"/>
+            <target type="serial" port="0"/>
+            <alias name="serial0"/>
+          </console>
+          <memballoon model="virtio">
+            <stats period="10"/>
+            <alias name="balloon0"/>
+            <address type="pci" domain="0x0000" bus="0x00" slot="0x06" function="0x0"/>
+          </memballoon>
+        </devices>
+        <seclabel type="dynamic" model="dac" relabel="yes">
+          <label>+42424:+104</label>
+          <imagelabel>+42424:+104</imagelabel>
+        </seclabel>
+      </domain>
+    volumeTemplate: |
+      {% set nodename = 'master-' + item|string %}
+      <volume>
+        <name>{{ nodename }}</name>
+        <allocation>0</allocation>
+        <capacity unit='G'>{{ flavors.master.rootSize }}</capacity>
+        <target>
+          <format type='qcow2'/>
+        </target>
+      </volume>
+  worker:
+    domainTemplate: |
+      {% set nodename = 'worker-' + item|string %}
+      <domain type="kvm">
+        <name>{{ nodename }}</name>
+        <uuid>{{ nodename | hash('md5') }}</uuid>
+        <metadata>
+          <vino:flavor>worker</vino:flavor>
+          <vino:creationTime>{{ ansible_date_time.date }}</vino:creationTime>
+        </metadata>
+        <memory unit="GiB">{{ flavors.worker.memory }}</memory>
+        {% if flavors.worker.hugepages is defined and flavors.worker.hugepages == true %}
+        <memoryBacking>
+          <hugepages>
+            <page size='1' unit='GiB' />
+          </hugepages>
+        </memoryBacking>
+        {% endif %}
+        <vcpu placement="static">{{ flavors.worker.vcpus }}</vcpu>
+        {% if node_core_map[nodename] is defined %}
+        # function to produce list of cpus, in same numa (controled by bool), state will need to be tracked via file on hypervisor host. gotpl psudo:
+        <cputune>
+          <shares>8192</shares>
+          {% for core in node_core_map[nodename] %}
+          <vcpupin vcpu="{{ loop.index0 }}" cpuset="{{ core }}"/>
+          {% endfor %}
+          <emulatorpin cpuset="{{ node_core_map[nodename]|join(',') }}"/>
+        </cputune>
+        {% endif %}
+        <resource>
+          <partition>/machine</partition>
+        </resource>
+        <os>
+          <type arch="x86_64" machine="pc-i440fx-xenial">hvm</type>
+          <boot dev="hd"/>
+        </os>
+        <features>
+          <acpi/>
+          <apic/>
+        </features>
+        <cpu mode="host-passthrough" />
+        <clock offset="utc">
+          <timer name="pit" tickpolicy="delay"/>
+          <timer name="rtc" tickpolicy="catchup"/>
+          <timer name="hpet" present="no"/>
+        </clock>
+        <on_poweroff>destroy</on_poweroff>
+        <on_reboot>restart</on_reboot>
+        <on_crash>destroy</on_crash>
+        <devices>
+          <emulator>/usr/bin/qemu-system-x86_64</emulator>
+
+          # for each disk requested
+          <disk type='volume' device='disk'>
+            <driver name="qemu" type="qcow2" cache="none" discard="unmap"/>
+            <source pool='vino-default' volume='{{ nodename }}'/>
+            <target dev='vde' bus='virtio'/>
+          </disk>
+
+          <controller type="usb" index="0" model="piix3-uhci">
+            <alias name="usb"/>
+            <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x2"/>
+          </controller>
+          <controller type="pci" index="0" model="pci-root">
+            <alias name="pci.0"/>
+          </controller>
+          <controller type="ide" index="0">
+            <alias name="ide"/>
+            <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
+          </controller>
+
+          # for each interface defined in vino, e.g.
+          <interface type='bridge'>
+            <mac address='52:54:00:83:e9:f9'/>
+            <source bridge='management'/>
+            <model type='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+          </interface>
+
+          <serial type="pty">
+            <source path="/dev/pts/3"/>
+            <log file="/var/lib/vino/instances/{{ nodename }}.console.log" append="off"/>
+            <target type="isa-serial" port="0">
+              <model name="isa-serial"/>
+            </target>
+            <alias name="serial0"/>
+          </serial>
+          <console type="pty" tty="/dev/pts/3">
+            <source path="/dev/pts/3"/>
+            <log file="/var/lib/vino/instances/{{ nodename }}.console.log" append="off"/>
+            <target type="serial" port="0"/>
+            <alias name="serial0"/>
+          </console>
+          <memballoon model="virtio">
+            <stats period="10"/>
+            <alias name="balloon0"/>
+            <address type="pci" domain="0x0000" bus="0x00" slot="0x06" function="0x0"/>
+          </memballoon>
+        </devices>
+        <seclabel type="dynamic" model="dac" relabel="yes">
+          <label>+42424:+104</label>
+          <imagelabel>+42424:+104</imagelabel>
+        </seclabel>
+      </domain>
+    volumeTemplate: |
+      {% set nodename = 'worker-' + item|string %}
+      <volume>
+        <name>{{ nodename }}</name>
+        <allocation>0</allocation>
+        <capacity unit='G'>{{ flavors.worker.rootSize }}</capacity>
+        <target>
+          <format type='qcow2'/>
+        </target>
+      </volume>

config/manager/flavors.yaml

@@ -0,0 +1,11 @@
+flavors:
+  master:
+    vcpus: 4
+    memory: 4
+    hugepages: false
+    rootSize: 30
+  worker:
+    vcpus: 2
+    memory: 2
+    hugepages: false
+    rootSize: 10

config/manager/kustomization.yaml

@@ -7,3 +7,23 @@ configMapGenerator:
     disableNameSuffixHash: true
   files:
   - template=daemonset-template.yaml
+- name: flavors
+  options:
+    disableNameSuffixHash: true
+  files:
+    - flavors.yaml
+- name: flavor-templates
+  options:
+    disableNameSuffixHash: true
+  files:
+    - flavor-templates.yaml
+- name: network-templates
+  options:
+    disableNameSuffixHash: true
+  files:
+    - network-templates.yaml
+- name: storage-templates
+  options:
+    disableNameSuffixHash: true
+  files:
+    - storage-templates.yaml

config/manager/network-templates.yaml

@@ -0,0 +1,17 @@
+libvirtNetworks:
+  - name: management
+    libvirtTemplate: |
+      <network>
+        <name>management</name>
+        <forward mode='route'/>
+        <bridge name='management' stp='off' delay='0'/>
+        <ip address='{{ networks[0].routes[0].gateway }}' netmask='255.255.240.0'>
+          <!-- <tftp root='/srv/tftp'/> -->
+          <dhcp>
+            <range start='{{ networks[0].allocationStart }}' end='{{ networks[0].allocationStop }}'/>
+            <bootp file=''/>
+          </dhcp>
+        </ip>
+      </network>
+#  - name: mobility-gn
+#    libvirtTemplate:

config/manager/storage-templates.yaml

@@ -0,0 +1,14 @@
+libvirtStorage:
+  - name: vino-default
+    libvirtTemplate: |
+      <pool type='dir'>
+        <name>vino-default</name>
+        <target>
+          <path>/var/lib/libvirt/vino</path>
+          <permissions>
+            <mode>0711</mode>
+            <owner>0</owner>
+            <group>0</group>
+          </permissions>
+        </target>
+      </pool>

config/samples/vino_cr.yaml

@@ -2,7 +2,7 @@ apiVersion: airship.airshipit.org/v1
 kind: Vino
 metadata:
   name: vino-test-cr
-  #  labels: ...
+  labels: {}
 spec:
   nodeLabelKeysToCopy:
     - "airshipit.org/server"
@@ -14,6 +14,7 @@ spec:
     cpuExclude: 0-4,54-60
     redfishCredentialSecret:
       name: redfishSecret
+      namespace: airship-system
   networks:
     - name: management
       subnet: 192.168.2.0/20
@@ -23,7 +24,7 @@ spec:
       routes:
         - network: 10.0.0.0
           netmask: 255.255.255.0
-          gateway: $vino.nodebridgegw # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
+          gateway: 192.168.2.1 # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
       dns_servers: ["135.188.34.124"]
     - name: external
       subnet: 169.0.0.0/24
@@ -35,16 +36,20 @@ spec:
       allocationStart: 169.0.0.10
       allocationStop: 169.0.0.254
       macPrefix: "0A:00:00:00:00:00"
-
   vmBridge: lo
   nodes:
-    - name: "worker"
+    - name: master
-      count: 3
+      count: 1
       bmhLabels:
-        airshipit.org/k8s-role: worker
+        airshipit.org/k8s-role: master
       networkDataTemplate:
         name: "test-template"
         namespace: "default"
+      labels:
+        vmFlavor: master
+      # libvirtTemplate:
+      #  name: libvirt-template-master
+      #  namespace: vino-system
       networkInterfaces:
         - name: management
           type: bridge
@@ -63,7 +68,50 @@ spec:
             bond_mode: 802.3ad
             bond_xmit_hash_policy: layer3+4
             bond_miimon: "100"
+      diskDrives:
+        - name: root
+          type: qcow2
+          path: /home/foobar/qemu.img
+          options:
+            sizeGb: 30
+            sparse: true
+    - name: worker
+      count: 4
+      bmhLabels:
+        airshipit.org/k8s-role: worker
+      networkDataTemplate:
+        name: "test-template"
+        namespace: "default"
+      labels:
+        vmFlavor: worker
+      # libvirtTemplate:
+      #  name: libvirt-template-worker
+      #  namespace: vino-system
+      networkInterfaces:
+        - name: management
+          type: bridge
+          network: management
+          mtu: 1500
+          options:
+            bridgeName: vminfra-bridge
+        - name: external
+          type: sriov-bond
+          network: external
+          mtu: 9100
+          options:
+            # this is an 'open-ended' set of k/v pairs, validation is perfomed by vino rather than crd schema.
+            pf: "[enp29s0f0,enp219s1f1]"
+            vlan: "100"
+            bond_mode: 802.3ad
+            bond_xmit_hash_policy: layer3+4
+            bond_miimon: "100"
+      diskDrives:
+        - name: root
+          type: qcow2
+          path: /home/foobar/qemu.img
+          options:
+            sizeGb: 10
+            sparse: true
   bmcCredentials:
-    username: "admin"
+    username: admin
-    password: "passw0rd"
+    password: passw0rd
-

docs/api/vino.md

@@ -796,7 +796,7 @@ string
 </em>
 </td>
 <td>
-<p>Parameter for Node master or worker-standard</p>
+<p>Parameter for Node master or worker</p>
 </td>
 </tr>
 <tr>
@@ -850,7 +850,7 @@ NamespacedName
 <code>diskDrives</code><br>
 <em>
 <a href="#airship.airshipit.org/v1.DiskDrivesTemplate">
-DiskDrivesTemplate
+[]DiskDrivesTemplate
 </a>
 </em>
 </td>

pkg/api/v1/vino_types.go

@@ -104,7 +104,7 @@ type VMRoutes struct {
 
 //NodeSet node definitions
 type NodeSet struct {
-	//Parameter for Node master or worker-standard
+	//Parameter for Node master or worker
 	Name  string `json:"name,omitempty"`
 	Count int    `json:"count,omitempty"`
 	// BMHLabels labels will be copied directly to BMHs that will be created
@@ -112,7 +112,7 @@ type NodeSet struct {
 	BMHLabels                 map[string]string    `json:"bmhLabels,omitempty"`
 	LibvirtTemplateDefinition NamespacedName       `json:"libvirtTemplate,omitempty"`
 	NetworkInterfaces         []NetworkInterface   `json:"networkInterfaces,omitempty"`
-	DiskDrives                *DiskDrivesTemplate `json:"diskDrives,omitempty"`
+	DiskDrives                []DiskDrivesTemplate `json:"diskDrives,omitempty"`
 	// NetworkDataTemplate must have a template key
 	NetworkDataTemplate NamespacedName `json:"networkDataTemplate,omitempty"`
 }

pkg/api/v1/zz_generated.deepcopy.go

@@ -324,8 +324,10 @@ func (in *NodeSet) DeepCopyInto(out *NodeSet) {
 	}
 	if in.DiskDrives != nil {
 		in, out := &in.DiskDrives, &out.DiskDrives
-		*out = new(DiskDrivesTemplate)
+		*out = make([]DiskDrivesTemplate, len(*in))
-		(*in).DeepCopyInto(*out)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	out.NetworkDataTemplate = in.NetworkDataTemplate
 }

pkg/controllers/vino_controller.go

@@ -320,7 +320,7 @@ func (r *VinoReconciler) ensureDaemonSet(ctx context.Context, vino *vinov1.Vino)
 	// controller should watch for changes in daemonset to reconcile if it breaks, and change status
 	// of the vino object
 	// controlleruti.SetControllerReference(vino, ds, r.scheme)
-	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+	ctx, cancel := context.WithTimeout(ctx, time.Second*180)
 	defer cancel()
 
 	return r.waitDaemonSet(ctx, ds)
@@ -440,7 +440,7 @@ func (r *VinoReconciler) waitDaemonSet(ctx context.Context, ds *appsv1.DaemonSet
 				}
 				logger.Info("DaemonSet is not in ready status, rechecking in 2 seconds")
 			}
-			time.Sleep(2 * time.Second)
+			time.Sleep(10 * time.Second)
 		}
 	}
 }

tools/deployment/test-cr.sh

@@ -14,7 +14,8 @@ function vinoDebugInfo () {
 
 server_label="airshipit.org/server=s1"
 rack_label="airshipit.org/rack=r1"
-copyLabel="airshipit.org/k8s-role=worker"
+master_copy_label="airshipit.org/k8s-role=master"
+worker_copy_label="airshipit.org/k8s-role=worker"
 
 # Label all nodes with the same rack/label. We are ok with this for this simple test.
 kubectl label node --overwrite=true --all $server_label $rack_label
@@ -34,7 +35,7 @@ until [[ $(kubectl get vino vino-test-cr 2>/dev/null) ]]; do
   fi
   sleep 2
 done
-if ! kubectl wait --for=condition=Ready vino vino-test-cr --timeout=180s; then
+if ! kubectl wait --for=condition=Ready vino vino-test-cr --timeout=600s; then
     vinoDebugInfo
 fi
 
@@ -52,11 +53,17 @@ if ! kubectl -n vino-system rollout status ds default-vino-test-cr --timeout=10s
     vinoDebugInfo
 fi
 
-bmhCount=$(kubectl get baremetalhosts -n vino-system -l "$server_label,$server_label,$copyLabel" -o name | wc -l)
+masterCount=$(kubectl get baremetalhosts -n vino-system -l "$server_label,$server_label,$master_copy_label" -o name | wc -l)
 
-# with this setup set up, exactly 3 BMHs must have been created by VINO controller
+# with this setup set up, exactly 1 master must have been created by VINO controller
 
-[[ "$bmhCount" -eq "3" ]]
+[[ "$masterCount" -eq "1" ]]
+
+workerCount=$(kubectl get baremetalhosts -n vino-system -l "$server_label,$server_label,$worker_copy_label" -o name | wc -l)
+
+# with this setup set up, exactly 4 workers must have been created by VINO controller
+
+[[ "$workerCount" -eq "4" ]]
 
 kubectl get baremetalhosts -n vino-system --show-labels=true