From a71ce32bf6ba7916e1baf004b0050df469107850 Mon Sep 17 00:00:00 2001
From: Andy Ning <andy.ning@windriver.com>
Date: Thu, 17 Jun 2021 20:58:24 -0400
Subject: [PATCH] Kube rootca update abort changes
This change updated k8s master and worker rootca runtime classes
to support kube rootca update abort procedure.
We also added a fix to cert-mon errors when update start again
after abort.
When root ca update start again after abort at the stage of host
certs get updated, cert-mon will generate TLS errors at the stage
of trustbothcas. This is fixed by moving the signing ca cert to
the top of the admin.conf when update-certs phase completes on the
host.
Story: 2008675
Task: 43067
Signed-off-by: Andy Ning <andy.ning@windriver.com>
Change-Id: I79d4298694bcba96b89602e93adca49a71d284b6
---
.../modules/platform/manifests/kubernetes.pp | 55 +++++++++----------
1 file changed, 26 insertions(+), 29 deletions(-)
diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
index 2afed2916..5ffef0772 100644
--- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
+++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
@@ -46,9 +46,6 @@ class platform::kubernetes::params (
$controller_manager_key = undef,
$kubelet_cert = undef,
$kubelet_key = undef,
- # The file holding the original root CA cert/key before update
- $rootca_certfile_old = '/etc/kubernetes/pki/ca_old.crt',
- $rootca_keyfile_old = '/etc/kubernetes/pki/ca_old.key',
# The file holding the root CA cert/key to update to
$rootca_certfile_new = '/etc/kubernetes/pki/ca_new.crt',
$rootca_keyfile_new = '/etc/kubernetes/pki/ca_new.key',
@@ -841,14 +838,8 @@ class platform::kubernetes::duplex_migration::runtime {
class platform::kubernetes::master::rootca::trustbothcas::runtime
inherits ::platform::kubernetes::params {
- # Backup the original root CA cert
- file { $rootca_certfile_old:
- ensure => file,
- source => $rootca_certfile,
- replace => false,
- }
# Create the new root CA cert file
- -> file { $rootca_certfile_new:
+ file { $rootca_certfile_new:
ensure => file,
content => base64('decode', $rootca_cert),
}
@@ -859,7 +850,7 @@ class platform::kubernetes::master::rootca::trustbothcas::runtime
}
# Append the new cert to the current cert
-> exec { 'append_ca_cert':
- command => "cat ${rootca_certfile_old} ${rootca_certfile_new} > ${rootca_certfile}",
+ command => "cat ${rootca_certfile_new} >> ${rootca_certfile}",
}
# update admin.conf with both old and new certs
-> exec { 'update_admin_conf':
@@ -909,16 +900,10 @@ class platform::kubernetes::master::rootca::trustbothcas::runtime
class platform::kubernetes::worker::rootca::trustbothcas::runtime
inherits ::platform::kubernetes::params {
- $cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \\
- | grep 'name:' | awk '{printf \"%s\", \$2}'")
- # Backup the original root CA cert
- file { $rootca_certfile_old:
- ensure => file,
- source => $rootca_certfile,
- replace => false,
- }
+ $cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \
+ | grep 'name:' | awk '{printf \"%s\", \$2}'")
# Create the new root CA cert file
- -> file { $rootca_certfile_new:
+ file { $rootca_certfile_new:
ensure => file,
content => base64('decode', $rootca_cert),
}
@@ -929,7 +914,8 @@ class platform::kubernetes::worker::rootca::trustbothcas::runtime
}
# Append the new cert to the current cert
-> exec { 'append_ca_cert':
- command => "cat ${rootca_certfile_old} ${rootca_certfile_new} > ${rootca_certfile}",
+ command => "cat ${rootca_certfile_new} >> ${rootca_certfile}",
+ unless => "grep -v '[BEGIN|END] CERTIFICATE' ${rootca_certfile_new} | awk /./ | grep -f ${rootca_certfile} &>/dev/null"
}
# Update kubelet.conf with both old and new certs
-> exec { 'update_kubelet_conf':
@@ -1007,10 +993,6 @@ class platform::kubernetes::master::rootca::trustnewca::runtime
-> exec { 'remove_new_key_file':
command => "/bin/rm -f ${rootca_keyfile_new}",
}
- # Remove the old cert file
- -> exec { 'remove_old_cert_file':
- command => "/bin/rm -f ${rootca_certfile_old}",
- }
}
class platform::kubernetes::worker::rootca::trustnewca::runtime
@@ -1028,10 +1010,6 @@ class platform::kubernetes::worker::rootca::trustnewca::runtime
command => "/bin/mv -f ${rootca_keyfile_new} ${rootca_keyfile}",
onlyif => "/usr/bin/test -e ${rootca_keyfile_new}",
}
- # Remove the old cert file
- -> exec { 'remove_old_cert_file':
- command => "/bin/rm -f ${rootca_certfile_old}",
- }
# Update kubelet.conf with the new cert
-> exec { 'update_kubelet_conf':
environment => [ 'KUBECONFIG=/etc/kubernetes/kubelet.conf' ],
@@ -1231,6 +1209,25 @@ class platform::kubernetes::master::rootca::updatecerts::runtime
command => "/usr/bin/kill -s SIGHUP $(pidof kubelet)"
}
+ # Moving the signing ca cert in ca.crt and admin.conf to be the first in the bundle.
+ # This is neccessary for cert-mon, since it only uses the first ca cert in admin.conf
+ # to verify server certificate from apiserver.
+ # Remove the new ca cert from the bundle first
+ -> exec { 'remove_new_ca_cert_from_bottom':
+ command => "tac ${rootca_certfile} | sed '0,/-----BEGIN CERTIFICATE-----/d' | tac - > /tmp/kube_rootca_update/ca_tmp.crt"
+ }
+
+ # Create the ca.crt with the new ca cert at the top of the bundle
+ -> exec { 'prepend_new_ca_cert_at_top':
+ command => "cat ${rootca_certfile_new} /tmp/kube_rootca_update/ca_tmp.crt > ${rootca_certfile}"
+ }
+
+ # Update admin.conf with the newly create ca.crt
+ -> exec { 'update_admin_conf_with_new_cert_at_top':
+ environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ],
+ command => "kubectl config set-cluster kubernetes --certificate-authority ${rootca_certfile} --embed-certs",
+ }
+
# Removing temporary directory for files along this configuration process
-> exec { 'remove_kube_rootca_update_dir':
command => '/usr/bin/rm -rf /tmp/kube_rootca_update',