2188b75a9d
This patchset moves vino spec from configMap to annotation. Also it utilizes vino builder API object. The object is not yet fully developed. In next steps will be to update vino builder, but it is in a separate repository right now, and isn't fully functional. Change-Id: Ifad74ebafe0e8444161549bab69a4c7f5a8ee58f
489 lines
14 KiB
Go
489 lines
14 KiB
Go
/*
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"text/template"
|
|
"time"
|
|
|
|
"github.com/Masterminds/sprig"
|
|
"github.com/go-logr/logr"
|
|
metal3 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
kerror "k8s.io/apimachinery/pkg/util/errors"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/yaml"
|
|
|
|
vinov1 "vino/pkg/api/v1"
|
|
"vino/pkg/ipam"
|
|
)
|
|
|
|
const (
|
|
// DefaultMACPrefix is a private RFC 1918 MAC range used if
|
|
// no MACPrefix is specified for a network in the ViNO CR
|
|
DefaultMACPrefix = "02:00:00:00:00:00"
|
|
)
|
|
|
|
type networkTemplateValues struct {
|
|
Node vinov1.NodeSet // the specific node type to be templated
|
|
BMHName string
|
|
Networks []vinov1.Network
|
|
Generated generatedValues // Host-specific values calculated by ViNO: IP, etc
|
|
}
|
|
|
|
type generatedValues struct {
|
|
IPAddresses map[string]string
|
|
MACAddresses map[string]string
|
|
}
|
|
|
|
func (r *VinoReconciler) ensureBMHs(ctx context.Context, vino *vinov1.Vino) error {
|
|
labelOpt := client.MatchingLabels{
|
|
vinov1.VinoLabelDSNameSelector: vino.Name,
|
|
vinov1.VinoLabelDSNamespaceSelector: vino.Namespace,
|
|
}
|
|
|
|
nsOpt := client.InNamespace(getRuntimeNamespace())
|
|
|
|
podList := &corev1.PodList{}
|
|
err := r.List(ctx, podList, labelOpt, nsOpt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
logger := logr.FromContext(ctx)
|
|
logger.Info("Vino daemonset pod count", "count", len(podList.Items))
|
|
|
|
for _, pod := range podList.Items {
|
|
logger.Info("Creating baremetal hosts for pod",
|
|
"pod name",
|
|
types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name},
|
|
)
|
|
err := r.createIpamNetworks(ctx, vino)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = r.createBMHperPod(ctx, vino, pod)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *VinoReconciler) reconcileBMHs(ctx context.Context, vino *vinov1.Vino) error {
|
|
if err := r.ensureBMHs(ctx, vino); err != nil {
|
|
err = fmt.Errorf("could not reconcile BaremetalHosts: %w", err)
|
|
apimeta.SetStatusCondition(&vino.Status.Conditions, metav1.Condition{
|
|
Status: metav1.ConditionFalse,
|
|
Reason: vinov1.ReconciliationFailedReason,
|
|
Message: err.Error(),
|
|
Type: vinov1.ConditionTypeReady,
|
|
ObservedGeneration: vino.GetGeneration(),
|
|
})
|
|
apimeta.SetStatusCondition(&vino.Status.Conditions, metav1.Condition{
|
|
Status: metav1.ConditionFalse,
|
|
Reason: vinov1.ReconciliationFailedReason,
|
|
Message: err.Error(),
|
|
Type: vinov1.ConditionTypeBMHReady,
|
|
ObservedGeneration: vino.GetGeneration(),
|
|
})
|
|
if patchStatusErr := r.patchStatus(ctx, vino); patchStatusErr != nil {
|
|
err = kerror.NewAggregate([]error{err, patchStatusErr})
|
|
err = fmt.Errorf("unable to patch status after BaremetalHosts reconciliation failed: %w", err)
|
|
}
|
|
return err
|
|
}
|
|
apimeta.SetStatusCondition(&vino.Status.Conditions, metav1.Condition{
|
|
Status: metav1.ConditionTrue,
|
|
Reason: vinov1.ReconciliationSucceededReason,
|
|
Message: "BaremetalHosts reconciled",
|
|
Type: vinov1.ConditionTypeBMHReady,
|
|
ObservedGeneration: vino.GetGeneration(),
|
|
})
|
|
if err := r.patchStatus(ctx, vino); err != nil {
|
|
err = fmt.Errorf("unable to patch status after BaremetalHosts reconciliation succeeded: %w", err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *VinoReconciler) createIpamNetworks(ctx context.Context, vino *vinov1.Vino) error {
|
|
logger := logr.FromContext(ctx)
|
|
for _, network := range vino.Spec.Networks {
|
|
subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
macPrefix := network.MACPrefix
|
|
if macPrefix == "" {
|
|
logger.Info("No MACPrefix provided; using default MACPrefix for network",
|
|
"default prefix", DefaultMACPrefix, "network name", network.Name)
|
|
macPrefix = DefaultMACPrefix
|
|
}
|
|
err = r.Ipam.AddSubnetRange(ctx, network.SubNet, subnetRange, macPrefix)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *VinoReconciler) createBMHperPod(ctx context.Context, vino *vinov1.Vino, pod corev1.Pod) error {
|
|
logger := logr.FromContext(ctx)
|
|
|
|
nodeNetworkValues := map[string]generatedValues{}
|
|
|
|
k8sNode, err := r.getNode(ctx, pod)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
nodeNetworks, err := r.nodeNetworks(ctx, vino.Spec.Networks, k8sNode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, node := range vino.Spec.Nodes {
|
|
logger.Info("Creating BMHs for vino node", "node name", node.Name, "count", node.Count)
|
|
prefix := r.getBMHNodePrefix(vino, pod)
|
|
for i := 0; i < node.Count; i++ {
|
|
roleSuffix := fmt.Sprintf("%s-%d", node.Name, i)
|
|
bmhName := fmt.Sprintf("%s-%s", prefix, roleSuffix)
|
|
|
|
creds, nodeErr := r.reconcileBMHCredentials(ctx, vino)
|
|
if nodeErr != nil {
|
|
return nodeErr
|
|
}
|
|
|
|
domainNetValues, nodeErr := r.domainSpecificNetValues(ctx, bmhName, node, nodeNetworks)
|
|
if nodeErr != nil {
|
|
return nodeErr
|
|
}
|
|
// save domain specific generated values to a map
|
|
nodeNetworkValues[roleSuffix] = domainNetValues.Generated
|
|
|
|
netData, netDataNs, nodeErr := r.reconcileBMHNetworkData(ctx, node, vino, domainNetValues)
|
|
if nodeErr != nil {
|
|
return nodeErr
|
|
}
|
|
|
|
bmcAddr, labels, nodeErr := r.getBMCAddressAndLabels(ctx, k8sNode, vino.Spec.NodeLabelKeysToCopy, roleSuffix)
|
|
if nodeErr != nil {
|
|
return nodeErr
|
|
}
|
|
|
|
for label, value := range node.BMHLabels {
|
|
labels[label] = value
|
|
}
|
|
|
|
bmh := &metal3.BareMetalHost{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: bmhName,
|
|
Namespace: getRuntimeNamespace(),
|
|
Labels: labels,
|
|
},
|
|
Spec: metal3.BareMetalHostSpec{
|
|
NetworkData: &corev1.SecretReference{
|
|
Name: netData,
|
|
Namespace: netDataNs,
|
|
},
|
|
BMC: metal3.BMCDetails{
|
|
Address: bmcAddr,
|
|
CredentialsName: creds,
|
|
DisableCertificateVerification: true,
|
|
},
|
|
},
|
|
}
|
|
objKey := client.ObjectKeyFromObject(bmh)
|
|
logger.Info("Creating BMH", "name", objKey)
|
|
nodeErr = applyRuntimeObject(ctx, objKey, bmh, r.Client)
|
|
if nodeErr != nil {
|
|
return nodeErr
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.Info("annotating node", "node", k8sNode.Name)
|
|
if err = r.annotateNode(ctx, k8sNode, nodeNetworkValues, vino); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// nodeNetworks returns a copy of node network with a unique per node values
|
|
func (r *VinoReconciler) nodeNetworks(ctx context.Context,
|
|
globalNetworks []vinov1.Network,
|
|
k8sNode *corev1.Node) ([]vinov1.Network, error) {
|
|
bridgeIP, err := r.getBridgeIP(ctx, k8sNode)
|
|
if err != nil {
|
|
return []vinov1.Network{}, err
|
|
}
|
|
|
|
for netIndex, network := range globalNetworks {
|
|
for routeIndex, route := range network.Routes {
|
|
if route.Gateway == "$vinobridge" {
|
|
globalNetworks[netIndex].Routes[routeIndex].Gateway = bridgeIP
|
|
}
|
|
}
|
|
}
|
|
return globalNetworks, nil
|
|
}
|
|
|
|
func (r *VinoReconciler) domainSpecificNetValues(
|
|
ctx context.Context,
|
|
bmhName string,
|
|
node vinov1.NodeSet,
|
|
networks []vinov1.Network) (networkTemplateValues, error) {
|
|
// Allocate an IP for each of this BMH's network interfaces
|
|
ipAddresses := map[string]string{}
|
|
macAddresses := map[string]string{}
|
|
for _, iface := range node.NetworkInterfaces {
|
|
networkName := iface.NetworkName
|
|
subnet := ""
|
|
var err error
|
|
subnetRange := vinov1.Range{}
|
|
for _, network := range networks {
|
|
if network.Name == networkName {
|
|
subnet = network.SubNet
|
|
subnetRange, err = ipam.NewRange(network.AllocationStart,
|
|
network.AllocationStop)
|
|
if err != nil {
|
|
return networkTemplateValues{}, err
|
|
}
|
|
break
|
|
}
|
|
}
|
|
if subnet == "" {
|
|
return networkTemplateValues{}, fmt.Errorf("Interface %s doesn't have a matching network defined", networkName)
|
|
}
|
|
ipAllocatedTo := fmt.Sprintf("%s/%s", bmhName, iface.NetworkName)
|
|
ipAddress, macAddress, err := r.Ipam.AllocateIP(ctx, subnet, subnetRange, ipAllocatedTo)
|
|
if err != nil {
|
|
return networkTemplateValues{}, err
|
|
}
|
|
ipAddresses[networkName] = ipAddress
|
|
macAddresses[iface.Name] = macAddress
|
|
logr.FromContext(ctx).Info("Got MAC and IP for the network and node",
|
|
"MAC", macAddress, "IP", ipAddress, "bmh name", bmhName)
|
|
}
|
|
return networkTemplateValues{
|
|
Node: node,
|
|
BMHName: bmhName,
|
|
Networks: networks,
|
|
Generated: generatedValues{
|
|
IPAddresses: ipAddresses,
|
|
MACAddresses: macAddresses,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func (r *VinoReconciler) annotateNode(ctx context.Context,
|
|
k8sNode *corev1.Node,
|
|
domainInterfaceValues map[string]generatedValues,
|
|
vino *vinov1.Vino) error {
|
|
logr.FromContext(ctx).Info("Getting GW bridge IP from node", "node", k8sNode.Name)
|
|
builderValues := vinov1.Builder{
|
|
Domains: make(map[string]vinov1.BuilderDomain),
|
|
Networks: vino.Spec.Networks,
|
|
Nodes: vino.Spec.Nodes,
|
|
CPUConfiguration: vino.Spec.CPUConfiguration,
|
|
}
|
|
for domainName, domain := range domainInterfaceValues {
|
|
builderDomain := vinov1.BuilderDomain{
|
|
Interfaces: make(map[string]vinov1.BuilderNetworkInterface),
|
|
}
|
|
for ifName, ifMAC := range domain.MACAddresses {
|
|
builderDomain.Interfaces[ifName] = vinov1.BuilderNetworkInterface{
|
|
MACAddress: ifMAC,
|
|
}
|
|
}
|
|
builderValues.Domains[domainName] = builderDomain
|
|
}
|
|
|
|
b, err := yaml.Marshal(builderValues)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
annotations := k8sNode.GetAnnotations()
|
|
if k8sNode.GetAnnotations() == nil {
|
|
annotations = make(map[string]string)
|
|
}
|
|
|
|
annotations[vinov1.VinoNodeNetworkValuesAnnotation] = string(b)
|
|
k8sNode.SetAnnotations(annotations)
|
|
|
|
return r.Update(ctx, k8sNode)
|
|
}
|
|
|
|
func (r *VinoReconciler) getBridgeIP(ctx context.Context, k8sNode *corev1.Node) (string, error) {
|
|
ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
|
|
for {
|
|
select {
|
|
case <-ctxTimeout.Done():
|
|
return "", ctx.Err()
|
|
default:
|
|
node := &corev1.Node{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: k8sNode.Name,
|
|
},
|
|
}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(node), node); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
ip, exist := k8sNode.Labels[vinov1.VinoDefaultGatewayBridgeLabel]
|
|
if exist {
|
|
return ip, nil
|
|
}
|
|
time.Sleep(10 * time.Second)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *VinoReconciler) getNode(ctx context.Context, pod corev1.Pod) (*corev1.Node, error) {
|
|
node := &corev1.Node{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: pod.Spec.NodeName,
|
|
},
|
|
}
|
|
err := r.Get(ctx, client.ObjectKeyFromObject(node), node)
|
|
return node, err
|
|
}
|
|
|
|
func (r *VinoReconciler) getBMHNodePrefix(vino *vinov1.Vino, pod corev1.Pod) string {
|
|
// TODO we need to do something about name length limitations
|
|
return fmt.Sprintf("%s-%s-%s", vino.Namespace, vino.Name, pod.Spec.NodeName)
|
|
}
|
|
|
|
func (r *VinoReconciler) getBMCAddressAndLabels(
|
|
ctx context.Context,
|
|
node *corev1.Node,
|
|
labelKeys []string,
|
|
vmName string) (string, map[string]string, error) {
|
|
logger := logr.FromContext(ctx).WithValues("k8s node", node.Name)
|
|
|
|
labels := map[string]string{}
|
|
|
|
for _, key := range labelKeys {
|
|
value, ok := node.Labels[key]
|
|
if !ok {
|
|
logger.Info("Kubernetes node missing label from vino CR CopyNodeLabelKeys field", "label", key)
|
|
}
|
|
labels[key] = value
|
|
}
|
|
|
|
for _, addr := range node.Status.Addresses {
|
|
if addr.Type == corev1.NodeInternalIP {
|
|
return fmt.Sprintf("redfish+http://%s:%d/redfish/v1/Systems/%s", addr.Address, 8000, vmName), labels, nil
|
|
}
|
|
}
|
|
return "", labels, fmt.Errorf("Node %s doesn't have internal ip address defined", node.Name)
|
|
}
|
|
|
|
// reconcileBMHCredentials returns secret name with credentials and error
|
|
func (r *VinoReconciler) reconcileBMHCredentials(ctx context.Context, vino *vinov1.Vino) (string, error) {
|
|
ns := getRuntimeNamespace()
|
|
// coresponds to DS name, since we have only one DS per vino CR
|
|
credentialSecretName := fmt.Sprintf("%s-%s", r.getDaemonSetName(vino), "credentials")
|
|
netSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: credentialSecretName,
|
|
Namespace: ns,
|
|
},
|
|
StringData: map[string]string{
|
|
"username": vino.Spec.BMCCredentials.Username,
|
|
"password": vino.Spec.BMCCredentials.Password,
|
|
},
|
|
Type: corev1.SecretTypeOpaque,
|
|
}
|
|
|
|
objKey := client.ObjectKeyFromObject(netSecret)
|
|
|
|
if err := applyRuntimeObject(ctx, objKey, netSecret, r.Client); err != nil {
|
|
return "", err
|
|
}
|
|
return credentialSecretName, nil
|
|
}
|
|
|
|
func (r *VinoReconciler) reconcileBMHNetworkData(
|
|
ctx context.Context,
|
|
node vinov1.NodeSet,
|
|
vino *vinov1.Vino,
|
|
values networkTemplateValues) (string, string, error) {
|
|
secret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: node.NetworkDataTemplate.Name,
|
|
Namespace: node.NetworkDataTemplate.Namespace,
|
|
},
|
|
}
|
|
|
|
logger := logr.FromContext(ctx).WithValues("vino node", node.Name, "vino", client.ObjectKeyFromObject(vino))
|
|
|
|
objKey := client.ObjectKeyFromObject(secret)
|
|
logger.Info("Looking for secret with network template for vino node", "secret", objKey)
|
|
if err := r.Get(ctx, objKey, secret); err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
rawTmpl, ok := secret.Data[TemplateDefaultKey]
|
|
if !ok {
|
|
return "", "", fmt.Errorf("network template secret %v has no key '%s'", objKey, TemplateDefaultKey)
|
|
}
|
|
|
|
tpl, err := template.New("net-template").Funcs(sprig.TxtFuncMap()).Parse(string(rawTmpl))
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
logger.Info("Genereated MAC Addresses values are", "GENERATED VALUES", values.Generated.MACAddresses)
|
|
|
|
buf := bytes.NewBuffer([]byte{})
|
|
err = tpl.Execute(buf, values)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
name := fmt.Sprintf("%s-network-data", values.BMHName)
|
|
ns := getRuntimeNamespace()
|
|
netSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: ns,
|
|
},
|
|
StringData: map[string]string{
|
|
"networkData": buf.String(),
|
|
},
|
|
Type: corev1.SecretTypeOpaque,
|
|
}
|
|
|
|
objKey = client.ObjectKeyFromObject(netSecret)
|
|
|
|
logger.Info("Creating network secret for vino node", "secret", objKey)
|
|
|
|
if err := applyRuntimeObject(ctx, objKey, netSecret, r.Client); err != nil {
|
|
return "", "", err
|
|
}
|
|
return name, ns, nil
|
|
}
|