Add Redfish support to Jump Host service

This change brings Redfish support to the Jump Host service, enabling
sub-cluster operators to reboot virtual machines in their sub-cluster.
With a Secret containing sub-cluster host information (e.g. BMC
addresses, BMC usernames, BMC password) and a ConfigMap containing a
wrapper script for DMTF's redfishtool, a user in a sub-cluster's Jump
Pod can manage their hosts by executing /sip/scripts/reboot [CMD].

Signed-off-by: Drew Walters <andrew.walters@att.com>
Change-Id: Iff71ad2287cb095ebe92445a4a09771697efa5ee

config/crd/bases/airship.airshipit.org_sipclusters.yaml

@@ -103,6 +103,12 @@ spec:
                     description: JumpHostService is an infrastructure service type
                       that represents the sub-cluster jump-host service.
                     properties:
+                      bmc:
+                        description: BMCOpts contains options for BMC communication.
+                        properties:
+                          proxy:
+                            type: boolean
+                        type: object
                       inline:
                         properties:
                           clusterIP:

config/rbac/sipcluster_scheduler_role.yaml

@@ -68,6 +68,7 @@ rules:
 - apiGroups:
   - ""
   resources:
+  - configmaps
   - services
   verbs:
   - create

config/samples/airship_v1beta1_sipcluster.yaml

@@ -28,16 +28,14 @@ spec:
         nodePort: 7023
         nodeInterfaceId: oam-ipv4
       - serviceType: jumphost
-        optional:
+        redfish:
-          sshKey: rsa.... #<-- this needs to align to the ssh keys provided to cluster api objects
+          proxy: false
-        image: ubuntu:20.04
+        image: quay.io/airshipit/jump-host:dev
         nodeLabels:
           - airship-control-plane
         nodePort: 7022
         nodeInterfaceId: oam-ipv4
       - serviceType: loadbalancer
-        optional:
-          clusterIP: 1.2.3.4 #<-- this aligns to the VIP IP for undercloud k8s
         image: haproxy:2.3.2
         nodeLabels:
           - airship-control-plane

docs/api/sipcluster.md

@@ -9,6 +9,37 @@
 <p>Package v1 contains API Schema definitions for the airship v1 API group</p>
 Resource Types:
 <ul class="simple"></ul>
+<h3 id="airship.airshipit.org/v1.BMCOpts">BMCOpts
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#airship.airshipit.org/v1.JumpHostService">JumpHostService</a>)
+</p>
+<p>BMCOpts contains options for BMC communication.</p>
+<div class="md-typeset__scrollwrap">
+<div class="md-typeset__table">
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>proxy</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+</td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
 <h3 id="airship.airshipit.org/v1.JumpHostService">JumpHostService
 </h3>
 <p>
@@ -40,6 +71,18 @@ SIPClusterService
 </tr>
 <tr>
 <td>
+<code>bmc</code><br>
+<em>
+<a href="#airship.airshipit.org/v1.BMCOpts">
+BMCOpts
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
+<tr>
+<td>
 <code>sshkey</code><br>
 <em>
 string

pkg/api/v1/sipcluster_types.go

@@ -86,7 +86,8 @@ func (s SIPClusterServices) GetAll() []SIPClusterService {
 // JumpHostService is an infrastructure service type that represents the sub-cluster jump-host service.
 type JumpHostService struct {
 	SIPClusterService `json:"inline"`
-	SSHKey            string `json:"sshkey,omitempty"`
+	BMC               *BMCOpts `json:"bmc,omitempty"`
+	SSHKey            string   `json:"sshkey,omitempty"`
 }
 
 // SIPClusterStatus defines the observed state of SIPCluster
@@ -165,6 +166,11 @@ type SIPClusterService struct {
 	ClusterIP     *string           `json:"clusterIP,omitempty"`
 }
 
+// BMCOpts contains options for BMC communication.
+type BMCOpts struct {
+	Proxy bool `json:"proxy,omitempty"`
+}
+
 // VMRole defines the states the provisioner will report
 // the tenant has having.
 type VMRole string

pkg/api/v1/zz_generated.deepcopy.go

@@ -25,10 +25,30 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *BMCOpts) DeepCopyInto(out *BMCOpts) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BMCOpts.
+func (in *BMCOpts) DeepCopy() *BMCOpts {
+	if in == nil {
+		return nil
+	}
+	out := new(BMCOpts)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *JumpHostService) DeepCopyInto(out *JumpHostService) {
 	*out = *in
 	in.SIPClusterService.DeepCopyInto(&out.SIPClusterService)
+	if in.BMC != nil {
+		in, out := &in.BMC, &out.BMC
+		*out = new(BMCOpts)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JumpHostService.

pkg/services/errors.go

@@ -26,5 +26,14 @@ type ErrInfraServiceNotSupported struct {
 }
 
 func (e ErrInfraServiceNotSupported) Error() string {
-	return fmt.Sprintf("Invalid Infrastructure Service: %v", e.Service)
+	return fmt.Sprintf("invalid Infrastructure Service: %v", e.Service)
+}
+
+// ErrMalformedRedfishAddress occurs when a Redfish address does not meet the expected format.
+type ErrMalformedRedfishAddress struct {
+	Address string
+}
+
+func (e ErrMalformedRedfishAddress) Error() string {
+	return fmt.Sprintf("invalid Redfish BMC address %s", e.Address)
 }

pkg/services/jumphost.go

@@ -15,6 +15,10 @@
 package services
 
 import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -28,9 +32,15 @@ import (
 
 const (
 	JumpHostServiceName = "jumphost"
+
+	mountPathData    = "/etc/opt/sip"
+	mountPathScripts = "/opt/sip/bin"
+
+	nameHostsVolume  = "hosts"
+	nameRebootVolume = "vm"
 )
 
-// JumpHost is an InfrastructureService that provides SSH capabilities to access a sub-cluster.
+// JumpHost is an InfrastructureService that provides SSH and power-management capabilities for sub-clusters.
 type jumpHost struct {
 	client   client.Client
 	sipName  types.NamespacedName
@@ -82,10 +92,75 @@ func (jh jumpHost) Deploy() error {
 		return err
 	}
 
+	// TODO: Validate Secret becomes ready.
+	secret, err := jh.generateSecret(instance, labels)
+	if err != nil {
+		return err
+	}
+
+	jh.logger.Info("Applying secret", "secret", secret.GetNamespace()+"/"+secret.GetName())
+	err = applyRuntimeObject(client.ObjectKey{Name: secret.GetName(), Namespace: secret.GetNamespace()},
+		secret, jh.client)
+	if err != nil {
+		return err
+	}
+
+	// TODO: Validate ConfigMap becomes ready.
+	configMap := jh.generateConfigMap(instance, labels)
+	jh.logger.Info("Applying configmap", "configmap", configMap.GetNamespace()+"/"+configMap.GetName())
+	err = applyRuntimeObject(client.ObjectKey{Name: configMap.GetName(), Namespace: configMap.GetNamespace()},
+		secret, jh.client)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
 func (jh jumpHost) generateDeployment(instance string, labels map[string]string) *appsv1.Deployment {
+	// NOTE(drewwalters96): the jump host container is declared here so environment variables can be easily added
+	// below based on configuration values in the SIPCluster CR.
+	jhContainer := corev1.Container{
+		Name:  JumpHostServiceName,
+		Image: jh.config.Image,
+		Ports: []corev1.ContainerPort{
+			{
+				Name:          "ssh",
+				ContainerPort: 22,
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      nameHostsVolume,
+				MountPath: mountPathData,
+			},
+			{
+				Name:      nameRebootVolume,
+				MountPath: mountPathScripts,
+			},
+		},
+		Command: []string{"/bin/sh"},
+		Args:    []string{"-c", "while true; do sleep 30; done"},
+	}
+
+	// Set NO_PROXY env variables when Redfish proxy setting is false (Default: false).
+	var proxy bool
+	if jh.config.BMC != nil {
+		proxy = jh.config.BMC.Proxy
+	}
+	if proxy == false {
+		jhContainer.Env = []corev1.EnvVar{
+			{
+				Name:  "NO_PROXY",
+				Value: "*",
+			},
+			{
+				Name:  "no_proxy",
+				Value: "*",
+			},
+		}
+	}
+
 	return &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      instance,
@@ -103,13 +178,25 @@ func (jh jumpHost) generateDeployment(instance string, labels map[string]string)
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
+						jhContainer,
+					},
+					Volumes: []corev1.Volume{
 						{
-							Name:  JumpHostServiceName,
+							Name: nameHostsVolume,
-							Image: jh.config.Image,
+							VolumeSource: corev1.VolumeSource{
-							Ports: []corev1.ContainerPort{
+								Secret: &corev1.SecretVolumeSource{
-								{
+									SecretName: instance,
-									Name:          "ssh",
+								},
-									ContainerPort: 22,
+							},
+						},
+						{
+							Name: nameRebootVolume,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: instance,
+									},
+									DefaultMode: int32Ptr(0777),
 								},
 							},
 						},
@@ -121,6 +208,45 @@ func (jh jumpHost) generateDeployment(instance string, labels map[string]string)
 	}
 }
 
+func (jh jumpHost) generateConfigMap(instance string, labels map[string]string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      instance,
+			Namespace: jh.sipName.Namespace,
+			Labels:    labels,
+		},
+		Data: map[string]string{
+			nameRebootVolume: fmt.Sprintf(rebootScript, mountPathData, nameHostsVolume),
+		},
+	}
+}
+
+func (jh jumpHost) generateSecret(instance string, labels map[string]string) (*corev1.Secret, error) {
+	hostData, err := generateHostList(*jh.machines)
+	if err != nil {
+		return nil, err
+	}
+
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      instance,
+			Namespace: jh.sipName.Namespace,
+			Labels:    labels,
+		},
+		Data: map[string][]byte{
+			nameHostsVolume: hostData,
+		},
+	}, nil
+}
+
 func (jh jumpHost) generateHostAliases() []corev1.HostAlias {
 	hostAliases := []corev1.HostAlias{}
 	for _, machine := range jh.machines.Machines {
@@ -166,6 +292,130 @@ func (jh jumpHost) Finalize() error {
 	return nil
 }
 
+type host struct {
+	Name string `json:"name"`
+	BMC  bmc    `json:"bmc"`
+}
+
+type bmc struct {
+	IP       string `json:"ip"`
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// generateHostList creates a list of hosts in JSON format to be mounted as a config map to the jump host pod and used
+// to power cycle sub-cluster nodes.
+func generateHostList(machineList airshipvms.MachineList) ([]byte, error) {
+	hosts := make([]host, len(machineList.Machines))
+	for name, machine := range machineList.Machines {
+		managementIP, err := getManagementIP(machine.BMH.Spec.BMC.Address)
+		if err != nil {
+			return nil, err
+		}
+
+		h := host{
+			Name: name,
+			BMC: bmc{
+				IP:       managementIP,
+				Username: machine.Data.BMCUsername,
+				Password: machine.Data.BMCPassword,
+			},
+		}
+
+		hosts = append(hosts, h)
+	}
+
+	out, err := json.Marshal(hosts)
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
+// getManagementIP parses the BMC IP address from a Redfish fully qualified domain name. For example, the input
+// redfish+https://127.0.0.1/redfish/v1/Systems/System.Embedded.1 yields 127.0.0.1.
+func getManagementIP(redfishURL string) (string, error) {
+	parsedURL, err := url.Parse(redfishURL)
+	if err != nil {
+		return "", ErrMalformedRedfishAddress{Address: redfishURL}
+	}
+
+	return parsedURL.Host, nil
+}
+
+var rebootScript = `#!/bin/sh
+
+# Support Infrastructure Provider (SIP) VM Utility
+# DO NOT MODIFY: generated by SIP
+
+HOSTS_FILE="%s/%s"
+
+LIST_COMMAND="list"
+REBOOT_COMMAND="reboot"
+
+help() {
+  echo "Support Infrastructure Provider (SIP) VM Utility"
+  echo ""
+  echo "Usage: ${LIST_COMMAND}                      list hosts"
+  echo "       ${REBOOT_COMMAND} [host name]        reboot host"
+}
+
+dep_check() {
+  if [ "$(which jq)" = "" ]; then
+    echo "Missing package 'jq'. Update your JumpHost image to include 'jq' and 'redfishtool'."
+    exit 1
+  fi
+
+  if [ "$(which redfishtool)" = "" ]; then
+    echo "Missing package 'redfishtool'. Update your JumpHost image to include 'jq' and 'redfishtool'."
+    exit 1
+  fi
+}
+
+get_bmc_info() {
+  for host in $(jq -r -c '.[]' ${HOSTS_FILE}); do
+    if [ "$(echo "$host" | jq -r '.name')" = "$1" ]; then
+      addr=$(echo "$host" | jq -r '.bmc.ip')
+      user=$(echo "$host" | jq -r '.bmc.username')
+      pass=$(echo "$host" | jq -r '.bmc.password')
+    fi
+  done
+}
+
+reboot() {
+  get_bmc_info "$1"
+  if [ "${addr}" = "" ] || [ "${user}" = "" ] || [ "$pass" = "" ]; then
+    echo "Invalid host '$1'. Use the '${LIST_COMMAND}' command to view hosts."
+    exit 1
+  fi
+
+  echo "Rebooting host '$1'"
+  redfishtool -r "${addr}" -u "${user}" -p "${pass}" \
+    Systems reset GracefulRestart -vvvvv
+  exit 0
+}
+
+case $1 in
+  "${LIST_COMMAND}")
+    dep_check
+    jq -r '.[].name' ${HOSTS_FILE}
+    ;;
+  "${REBOOT_COMMAND}")
+    if [ "$2" = "" ]; then
+      printf "Host name required.\n\n"
+      help
+      exit 1
+    fi
+    dep_check
+    reboot "$2"
+    ;;
+  *)
+    help
+    ;;
+esac
+`
+
 /*
 
 The SIP Cluster operator will manufacture a jump host pod specifically for this

pkg/services/services_test.go

@@ -2,6 +2,7 @@ package services_test
 
 import (
 	"context"
+	"encoding/json"
 
 	airshipv1 "sipcluster/pkg/api/v1"
 
@@ -25,6 +26,18 @@ const (
 var bmh1 *metal3.BareMetalHost
 var bmh2 *metal3.BareMetalHost
 
+// Re-declared from services package for testing purposes
+type host struct {
+	Name string `json:"name"`
+	BMC  bmc    `json:"bmc"`
+}
+
+type bmc struct {
+	IP       string `json:"ip"`
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
 var _ = Describe("Service Set", func() {
 	Context("When new SIP cluster is created", func() {
 		It("Deploys services", func() {
@@ -32,6 +45,16 @@ var _ = Describe("Service Set", func() {
 
 			bmh1, _ = testutil.CreateBMH(1, "default", "control-plane", 1)
 			bmh2, _ = testutil.CreateBMH(2, "default", "control-plane", 2)
+
+			bmcUsername := "root"
+			bmcPassword := "password"
+			bmcSecret := testutil.CreateBMCAuthSecret(bmh1.GetName(), bmh1.GetNamespace(), bmcUsername,
+				bmcPassword)
+			Expect(k8sClient.Create(context.Background(), bmcSecret)).Should(Succeed())
+
+			bmh1.Spec.BMC.CredentialsName = bmcSecret.Name
+			bmh2.Spec.BMC.CredentialsName = bmcSecret.Name
+
 			m1 := &vbmh.Machine{
 				BMH: *bmh1,
 				Data: &vbmh.MachineData{
@@ -40,6 +63,7 @@ var _ = Describe("Service Set", func() {
 					},
 				},
 			}
+
 			m2 := &vbmh.Machine{
 				BMH: *bmh2,
 				Data: &vbmh.MachineData{
@@ -68,13 +92,13 @@ var _ = Describe("Service Set", func() {
 			}
 
 			Eventually(func() error {
-				return testDeployment(sip)
+				return testDeployment(sip, *machineList)
 			}, 5, 1).Should(Succeed())
 		})
 	})
 })
 
-func testDeployment(sip *airshipv1.SIPCluster) error {
+func testDeployment(sip *airshipv1.SIPCluster, machineList vbmh.MachineList) error {
 	loadBalancerDeployment := &appsv1.Deployment{}
 	err := k8sClient.Get(context.Background(), types.NamespacedName{
 		Namespace: "default",
@@ -110,6 +134,7 @@ func testDeployment(sip *airshipv1.SIPCluster) error {
 	if err != nil {
 		return err
 	}
+
 	jumpHostHostAliases := jumpHostDeployment.Spec.Template.Spec.HostAliases
 	Expect(jumpHostHostAliases).To(ConsistOf(
 		corev1.HostAlias{
@@ -131,5 +156,26 @@ func testDeployment(sip *airshipv1.SIPCluster) error {
 		return err
 	}
 
+	jumpHostSecret := &corev1.Secret{}
+	err = k8sClient.Get(context.Background(), types.NamespacedName{
+		Namespace: "default",
+		Name:      services.JumpHostServiceName + "-" + sip.GetName(),
+	}, jumpHostSecret)
+	if err != nil {
+		return err
+	}
+
+	var hosts []host
+	err = json.Unmarshal(jumpHostSecret.Data["hosts"], &hosts)
+	Expect(err).To(BeNil())
+	for _, host := range hosts {
+		for _, machine := range machineList.Machines {
+			if host.Name == machine.BMH.Name {
+				Expect(host.BMC.Username).To(Equal(machine.Data.BMCUsername))
+				Expect(host.BMC.Password).To(Equal(machine.Data.BMCPassword))
+			}
+		}
+	}
+
 	return nil
 }

testutil/testutil.go

@@ -190,6 +190,9 @@ func CreateBMH(node int, namespace string, role airshipv1.VMRole, rack int) (*me
 					Namespace: namespace,
 					Name:      networkDataName,
 				},
+				BMC: metal3.BMCDetails{
+					Address: "redfish+https://32.68.51.12/redfish/v1/Systems/System.Embedded.1",
+				},
 			},
 		}, &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{