Conversion of storage during application update
Add lifecycle code to read secrets from PVC mounted to running
vault-manager, and vault-manager code for conversion of storage from PVC
to k8s secrets.
The lifecycle code is added because the previous version of
vault-manager does not respond to SIGTERM from kubernetes for
termination. And yet the pod will be terminating when the new
vault-manager pod runs. Reading the PVC data in lifecycle code before
helm updates the charts simplifies the process when vault-manager is
running during application-update.
The new vault-manager also handles the case where the application is not
running at the time the application is updated, such as if the
application is removed, deleted, uploaded and applied.
In general the procedure for conversion of the storage from PVC to k8s
secrets is:
- read the data from PVC
- store the data in k8s secrets
- validate the data
- confirm the stored data is the same as what was in PVC
- delete the original data only when the copy is confirmed
The solution employs a 'mount-helper', an incarnation of init.sh,
that mounts the PVC resource so that vault-manager can read it. The
mount-helper mounts the PVC resource and waits to be terminated.
Test plan:
PASS vault sanity
PASS vault sanity via application-update
PASS vault sanity update via application remove, delete, upload, apply
(update testing requires version bump similar to change 881754)
PASS unit test of the code
PASS bashate, flake8, bandit
PASS tox
Story: 2010930
Task: 48846
Change-Id: Iace37dad256b50f8d2ea6741bca070b97ec7d2d2
Signed-off-by: Michel Thebeau <Michel.Thebeau@windriver.com>
This commit is contained in:
Michel Thebeau
parent
cd165b8f5c
commit
464f9d0e76
@ -17,3 +17,5 @@ HELM_VAULT_MANAGER_POD = 'manager'
|
||||
HELM_VAULT_INJECTOR_POD = 'injector'
|
||||
|
||||
HELM_CHART_COMPONENT_LABEL = 'app.starlingx.io/component'
|
||||
|
||||
KEYSHARDS = 5
|
||||
|
||||
@ -0,0 +1,278 @@
|
||||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# All Rights Reserved.
|
||||
#
|
||||
|
||||
""" System inventory App lifecycle operator."""
|
||||
|
||||
from base64 import b64encode
|
||||
import json
|
||||
from k8sapp_vault.common import constants as app_constants
|
||||
from oslo_log import log as logging
|
||||
from sysinv.common import constants
|
||||
from sysinv.common import kubernetes
|
||||
from sysinv.common import utils as cutils
|
||||
from sysinv.helm import lifecycle_base as base
|
||||
import time
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
CONF = kubernetes.KUBERNETES_ADMIN_CONF
|
||||
NS = app_constants.HELM_CHART_NS_VAULT
|
||||
|
||||
# wait parameters for kubernetes secret creation
|
||||
WAIT_INTERVAL = 1 # seconds
|
||||
WAIT_COUNT = 10
|
||||
|
||||
|
||||
class VaultAppLifecycleOperator(base.AppLifecycleOperator):
|
||||
"""Lifecycle operator for vault application"""
|
||||
|
||||
def app_lifecycle_actions(self, context, conductor_obj, app_op, app, hook_info):
|
||||
"""Perform lifecycle actions for an operation
|
||||
|
||||
:param context: request context, can be None
|
||||
:param conductor_obj: conductor object, can be None
|
||||
:param app_op: AppOperator object
|
||||
:param app: AppOperator.Application object
|
||||
:param hook_info: LifecycleHookInfo object
|
||||
|
||||
"""
|
||||
if (hook_info.lifecycle_type == constants.APP_LIFECYCLE_TYPE_RESOURCE
|
||||
and hook_info.operation == constants.APP_APPLY_OP
|
||||
and hook_info.relative_timing == constants.APP_LIFECYCLE_TIMING_PRE):
|
||||
try:
|
||||
self.read_pvc_secret(app_op._kube._get_kubernetesclient_core())
|
||||
except Exception: # nosec # pylint: disable=broad-exception-caught
|
||||
# omit printing all exceptions in case any may
|
||||
# contain secret data
|
||||
pass
|
||||
|
||||
super().app_lifecycle_actions(context, conductor_obj, app_op,
|
||||
app, hook_info)
|
||||
|
||||
def validate_document(self, jdoc):
|
||||
"""Check whether the keyshards json is expected"""
|
||||
error = False
|
||||
|
||||
if not isinstance(jdoc, dict):
|
||||
LOG.error("document is not dict type")
|
||||
return False
|
||||
if not len(jdoc) == 3:
|
||||
LOG.error("document is not length 3")
|
||||
error = True
|
||||
|
||||
if 'keys' not in jdoc.keys():
|
||||
LOG.error("keys not in document")
|
||||
error = True
|
||||
elif not isinstance(jdoc['keys'], list):
|
||||
LOG.error("keys is not list type")
|
||||
error = True
|
||||
elif not len(jdoc['keys']) == app_constants.KEYSHARDS:
|
||||
LOG.error("len(keys) not expected")
|
||||
error = True
|
||||
|
||||
if 'keys_base64' not in jdoc.keys():
|
||||
LOG.error("keys_base64 not in document")
|
||||
error = True
|
||||
elif not isinstance(jdoc['keys_base64'], list):
|
||||
LOG.error("keys_base64 is not list type")
|
||||
error = True
|
||||
elif not len(jdoc['keys_base64']) == app_constants.KEYSHARDS:
|
||||
LOG.error("len(keys_base64) not expected")
|
||||
error = True
|
||||
|
||||
if 'root_token' not in jdoc.keys():
|
||||
LOG.error("root_token not in document")
|
||||
error = True
|
||||
elif not isinstance(jdoc['root_token'], str):
|
||||
LOG.error("root_token not str type")
|
||||
error = True
|
||||
|
||||
return not error
|
||||
|
||||
def ns_exists(self):
|
||||
"""check if vault is listed in namespaces"""
|
||||
jsonpath = '{.items[*].metadata.name}'
|
||||
cmd = ['kubectl', '--kubeconfig', CONF,
|
||||
'get', 'ns', '-o', 'jsonpath=' + jsonpath]
|
||||
stdout, stderr = cutils.trycmd(*cmd)
|
||||
if not stdout:
|
||||
LOG.info('Failed to get namespaces [%s]', stderr)
|
||||
return False
|
||||
if NS not in stdout.split():
|
||||
LOG.info('No vault namespace')
|
||||
return False
|
||||
return True
|
||||
|
||||
def get_pod_list(self):
|
||||
"""Get all pods in vault ns"""
|
||||
|
||||
if not self.ns_exists():
|
||||
return []
|
||||
|
||||
jsonpath = '{.items[*].metadata.name}'
|
||||
cmd = ['kubectl', '--kubeconfig', CONF,
|
||||
'get', 'pods', '-n', NS,
|
||||
'-o', 'jsonpath=' + jsonpath]
|
||||
stdout, stderr = cutils.trycmd(*cmd)
|
||||
if not stdout:
|
||||
LOG.info('No pods in vault namespace: [%s]', stderr)
|
||||
return []
|
||||
|
||||
return stdout.split()
|
||||
|
||||
def get_manager_pods(self):
|
||||
"""Get all pods named sva-vault-manager"""
|
||||
|
||||
managers = []
|
||||
pods = self.get_pod_list()
|
||||
for pod in pods:
|
||||
if pod.startswith('sva-vault-manager'):
|
||||
managers.append(pod)
|
||||
if not managers:
|
||||
LOG.info('failed to get vault-manager pod')
|
||||
return []
|
||||
return managers
|
||||
|
||||
def get_manager_pod(self):
|
||||
"""Return pod name if it has PVC mounted"""
|
||||
pods = self.get_manager_pods()
|
||||
|
||||
# assert that the vault-manager pod has PVC mounted
|
||||
managerpod = ''
|
||||
cspec = ".spec.containers[?(@.name=='manager')]"
|
||||
vspec = "volumeMounts[?(@.name=='manager-pvc')].name"
|
||||
jsonpath = "{%s.%s}" % (cspec, vspec)
|
||||
for pod in pods:
|
||||
cmd = ['kubectl', '--kubeconfig', CONF,
|
||||
'get', 'pods', '-n', NS, pod, '-o',
|
||||
'jsonpath=' + jsonpath]
|
||||
stdout, stderr = cutils.trycmd(*cmd)
|
||||
if stderr or not stdout:
|
||||
LOG.debug('vault-manager pod without PVC mounted'
|
||||
'[%s]', stderr)
|
||||
continue
|
||||
if managerpod:
|
||||
LOG.info('More than one vault-manager pod with PVC mounted'
|
||||
'[%s] and [%s]', managerpod, pod)
|
||||
managerpod = pod
|
||||
LOG.info('vault-manager pod with PVC mounted:'
|
||||
'[%s]', managerpod)
|
||||
return managerpod
|
||||
|
||||
def get_key_shards(self, podname):
|
||||
"""Read the key shards from vault-manager pod"""
|
||||
|
||||
cmd = ['kubectl', 'exec', '-n', NS, podname,
|
||||
'--kubeconfig', CONF,
|
||||
'--', 'cat', '/mnt/data/cluster_keys.json']
|
||||
stdout, stderr = cutils.trycmd(*cmd)
|
||||
if stderr or not stdout:
|
||||
LOG.info('cluster keys missing from PVC storage')
|
||||
return ''
|
||||
return stdout.strip()
|
||||
|
||||
def create_secret(self, client, shards):
|
||||
"""create a secret from shards text"""
|
||||
metadata = {'name': 'cluster-key-bootstrap'}
|
||||
data = {'strdata': shards}
|
||||
body = {'apiVersion': 'v1',
|
||||
'metadata': metadata,
|
||||
'data': data,
|
||||
'kind': 'Secret'}
|
||||
try:
|
||||
api_response = client.create_namespaced_secret(NS, body)
|
||||
except kubernetes.client.exceptions.ApiException:
|
||||
# omitting printing the exception text in case it may
|
||||
# contain the secrets content
|
||||
LOG.error('Failed to create bootstrap secret '
|
||||
'(ApiException)')
|
||||
return False
|
||||
|
||||
# verify that the api response contains the secret
|
||||
if ('data' in dir(api_response)
|
||||
and 'strdata' in api_response.data
|
||||
and api_response.data['strdata'] == shards):
|
||||
LOG.info('API response includes correct data')
|
||||
else:
|
||||
LOG.error('Failed to verify kubernetes api response')
|
||||
|
||||
# Ignore the above verification and continue
|
||||
return True
|
||||
|
||||
def read_pvc_secret(self, client):
|
||||
"""Retrieve key shards from a running vault-manager pod
|
||||
|
||||
The key shards are stored into k8s secrete
|
||||
'cluster-key-bootstrap', to be consumed by the new vault-manager
|
||||
pod. The vault-manager will also delete the PVC resource after
|
||||
successful validations.
|
||||
|
||||
Do nothing if:
|
||||
- no vault-manager pod is running with PVC attached (i.e.: no
|
||||
namespace, no pod. no vault-manager or no PVC attached)
|
||||
- PVC does not contain the expected key shards file
|
||||
- key shards data is not in an expected format (data structure,
|
||||
number of key shards
|
||||
|
||||
Print only soft errors if the validation of stored k8s secret
|
||||
is not successful.
|
||||
"""
|
||||
|
||||
podname = self.get_manager_pod()
|
||||
if not podname:
|
||||
LOG.info('No vault-manager with PVC mounted')
|
||||
return
|
||||
|
||||
keyshards = self.get_key_shards(podname)
|
||||
if not keyshards:
|
||||
# an error is printed
|
||||
return
|
||||
|
||||
# using encode()/decode() because b64encode requires bytes, but
|
||||
# kubernetes api requires str
|
||||
b64_keyshards = b64encode(keyshards.encode()).decode()
|
||||
|
||||
# assert that it's a json document with expected keys
|
||||
try:
|
||||
document = json.loads(keyshards)
|
||||
except json.decoder.JSONDecodeError:
|
||||
LOG.error("Failed to parse json document")
|
||||
return
|
||||
|
||||
if self.validate_document(document):
|
||||
LOG.info("Successfully retrieved %s key shards",
|
||||
len(document['keys']))
|
||||
else:
|
||||
LOG.error('The data appears invalid')
|
||||
return
|
||||
|
||||
if not self.create_secret(client, b64_keyshards):
|
||||
# an error is already printed
|
||||
return
|
||||
|
||||
# read the secret back
|
||||
LOG.info('Wait for the secret to be created')
|
||||
count = WAIT_COUNT
|
||||
while count > 0:
|
||||
# read the secret back
|
||||
jsonpath = "{.data.strdata}"
|
||||
cmd = ['kubectl', '--kubeconfig', CONF,
|
||||
'get', 'secrets', '-n', NS, 'cluster-key-bootstrap',
|
||||
'-o', 'jsonpath=' + jsonpath]
|
||||
stdout, stderr = cutils.trycmd(*cmd)
|
||||
if stdout and b64_keyshards == stdout:
|
||||
break
|
||||
LOG.debug('Result kubectl get secret '
|
||||
'cluster-key-bootstrap: [%s]', stderr)
|
||||
count -= 1
|
||||
time.sleep(WAIT_INTERVAL)
|
||||
|
||||
if b64_keyshards == stdout:
|
||||
LOG.info('Validation of stored key shards successful')
|
||||
else:
|
||||
LOG.error('Validation of stored key shards failed')
|
||||
@ -34,5 +34,9 @@ systemconfig.helm_applications =
|
||||
|
||||
systemconfig.helm_plugins.vault =
|
||||
001_vault = k8sapp_vault.helm.vault:VaultHelm
|
||||
|
||||
systemconfig.app_lifecycle =
|
||||
vault = k8sapp_vault.lifecycle.lifecycle_vault:VaultAppLifecycleOperator
|
||||
|
||||
[bdist_wheel]
|
||||
universal = 1
|
||||
|
||||
@ -31,6 +31,22 @@ data:
|
||||
# reserve trap '0' for disabling a debugging trap request
|
||||
DEBUGGING_TRAP=0
|
||||
|
||||
# Maximum sleep seconds for mount-helper before exiting
|
||||
MOUNT_HELPER_MAX_TIME=60
|
||||
|
||||
# Maximum seconds to wait for mount-helper pod to start
|
||||
MAX_POD_RUN_TRIES=10
|
||||
|
||||
# Maximum seconds to wait for vault-manager pod to exit
|
||||
# Vault-manager is not responding to SIGTERM, so will take 30
|
||||
# seconds
|
||||
TERMINATE_TRIES_MAX=6
|
||||
TERMINATE_TRIES_SLEEP=5
|
||||
|
||||
# Vault key share configuration
|
||||
KEY_SECRET_SHARES=5
|
||||
KEY_REQUIRED_THRESHOLD=3
|
||||
|
||||
# Records for seal status state machine:
|
||||
PODREC_F="$WORKDIR/previous_pods_status.txt"
|
||||
PODREC_TMP_F="$WORKDIR/new_pods_status.txt"
|
||||
@ -70,9 +86,6 @@ data:
|
||||
LOG_LEVEL={{ .Values.manager.log.defaultLogLevel }}
|
||||
LOG_OVERRIDE_FILE="$WORKDIR/log_level"
|
||||
|
||||
# Number of key shards
|
||||
KEY_SECRET_SHARES=5
|
||||
|
||||
# FUNCTIONS
|
||||
|
||||
# Convert log level to text for log message
|
||||
@ -217,7 +230,10 @@ data:
|
||||
rm -f "$WORKDIR"/s1 "$WORKDIR"/s2
|
||||
}
|
||||
|
||||
# validation function for splitShard
|
||||
# Check the structure of json data and confirm equivalence of
|
||||
# the stdin with stored secrets
|
||||
#
|
||||
# Returns the normal linux success=0, failure!=0
|
||||
function validateSecrets {
|
||||
local index
|
||||
local text
|
||||
@ -234,6 +250,8 @@ data:
|
||||
keys=$( echo "$text" | jq '.keys' )
|
||||
keys_base64=$( echo "$text" | jq '.keys_base64' )
|
||||
root_token=$( echo "$text" | jq -r '.root_token' )
|
||||
# response is 'null' if the dict key is missing
|
||||
# response is empty (-z) is the source document is empty
|
||||
if [ -z "$keys" -o "$keys" == "null" \
|
||||
-o -z "$keys_base64" -o "$keys_base64" == "null" \
|
||||
-o -z "$root_token" -o "$root_token" == "null" ]; then
|
||||
@ -242,14 +260,22 @@ data:
|
||||
fi
|
||||
|
||||
count=$( echo "$keys" | jq '. | length' )
|
||||
if [ $? -ne 0 ]; then
|
||||
log $ERROR "jq did not parse keys length"
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$count" ] || [ "$count" -ne "$KEY_SECRET_SHARES" ]; then
|
||||
log $ERROR "Incorrect array length for keys: " \
|
||||
log $ERROR "Incorrect array length for keys:" \
|
||||
"$count instead of $KEY_SECRET_SHARES"
|
||||
return 1
|
||||
fi
|
||||
count=$( echo "$keys_base64" | jq '. | length' )
|
||||
if [ $? -ne 0 ]; then
|
||||
log $ERROR "jq did not parse keys_base64 length"
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$count" ] || [ "$count" -ne "$KEY_SECRET_SHARES" ]; then
|
||||
log $ERROR "Incorrect array length for keys_base64: " \
|
||||
log $ERROR "Incorrect array length for keys_base64:" \
|
||||
"$count instead of $KEY_SECRET_SHARES"
|
||||
return 1
|
||||
fi
|
||||
@ -267,6 +293,9 @@ data:
|
||||
root_saved="$( get_secret cluster-key-root )"
|
||||
saved=$( echo "$saved" | jq \
|
||||
-c '{keys: .keys, keys_base64: .keys_base64, root_token: "'$root_saved'"}' )
|
||||
|
||||
# finally ensure that the saved secrets are the same as the
|
||||
# supplied text
|
||||
shaA=$( echo "$text" | sha256sum )
|
||||
shaB=$( echo "$saved" | sha256sum )
|
||||
if [ "$shaA" != "$shaB" ]; then
|
||||
@ -274,7 +303,7 @@ data:
|
||||
return 1
|
||||
fi
|
||||
|
||||
log $INFO "Verified creation of secret"
|
||||
log $INFO "Verified stored secrets are the same as supplied data"
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -323,25 +352,44 @@ data:
|
||||
done
|
||||
}
|
||||
|
||||
# Takes the json document output from vault initialization
|
||||
# and stores it into secrets for key shards and the root token
|
||||
#
|
||||
# This only works if the secrets are not pre-existing. An error
|
||||
# is printed by set_secrets.
|
||||
function storeVaultInitSecrets {
|
||||
local secrets="$1"
|
||||
local index
|
||||
local split_json
|
||||
|
||||
for index in $(seq 0 $((KEY_SECRET_SHARES - 1 ))); do
|
||||
split_json=$( echo -n "$secrets" | splitShard "$index" )
|
||||
set_secret "cluster-key-$index" /dev/stdin <<< "$split_json"
|
||||
done
|
||||
|
||||
split_json=$( echo "$secrets" | jq -r '.root_token' )
|
||||
set_secret "cluster-key-root" /dev/stdin <<< "$split_json"
|
||||
}
|
||||
|
||||
# Initializes the first vault pod, only needs to be performed once
|
||||
# after deploying the helm chart
|
||||
# Stores the root token and master key shards in k8s secrets
|
||||
function initVault {
|
||||
local V0
|
||||
local keys
|
||||
local index
|
||||
local split_json
|
||||
local secret_json
|
||||
local key_error
|
||||
local shares
|
||||
local threshold
|
||||
|
||||
V0=$(awk 'NR==1{print $2}' $WORKDIR/pods.txt)
|
||||
log $INFO "Initializing $V0"
|
||||
secret_json="{\"secret_shares\": $KEY_SECRET_SHARES, \"secret_threshold\": 3}"
|
||||
|
||||
shares='"secret_shares": '$KEY_SECRET_SHARES
|
||||
threshold='"secret_threshold": '$KEY_REQUIRED_THRESHOLD
|
||||
keys=$(
|
||||
curl -s \
|
||||
--cacert $CERT \
|
||||
--request POST \
|
||||
--data "$secret_json" \
|
||||
--data "{$shares, $threshold}" \
|
||||
https://$V0.$DOMAIN:8200/v1/sys/init
|
||||
)
|
||||
key_error=$(echo -n "$keys"| jq -r '.errors[]?')
|
||||
@ -349,15 +397,9 @@ data:
|
||||
log $ERROR "vault init request failed: $key_error"
|
||||
fi
|
||||
|
||||
for index in $(seq 0 $((KEY_SECRET_SHARES - 1 ))); do
|
||||
split_json=$( echo -n "$keys" | splitShard "$index" )
|
||||
set_secret "cluster-key-$index" /dev/stdin <<< "$split_json"
|
||||
done
|
||||
storeVaultInitSecrets "$keys"
|
||||
|
||||
split_json=$( echo "$keys" | jq -r '.root_token' )
|
||||
set_secret "cluster-key-root" /dev/stdin <<< "$split_json"
|
||||
|
||||
# validation if the split and store secrets is identical to the original pulled.
|
||||
# check if the secrets match vault's REST API response
|
||||
echo "$keys" | validateSecrets
|
||||
}
|
||||
|
||||
@ -506,9 +548,17 @@ data:
|
||||
function set_secret {
|
||||
local secret="$1"
|
||||
local contentf="$2"
|
||||
local output
|
||||
local result
|
||||
|
||||
kubectl create secret generic -n "$VAULT_NS" "$secret" \
|
||||
"--from-file=strdata=$contentf"
|
||||
output="$( kubectl create secret generic -n "$VAULT_NS" \
|
||||
"$secret" "--from-file=strdata=$contentf" 2>&1 )"
|
||||
result=$?
|
||||
if [ "$result" -ne 0 ]; then
|
||||
log $ERROR "Failed to create secret $secret"
|
||||
log $DEBUG "Output: [$output]"
|
||||
fi
|
||||
return $result
|
||||
}
|
||||
|
||||
function get_secret {
|
||||
@ -519,11 +569,410 @@ data:
|
||||
| base64 -d
|
||||
}
|
||||
|
||||
# When vault-manager is run in "MOUNT_HELPER" mode, this function
|
||||
# will not return. Instead the function will exit_on_trap or exit
|
||||
# when it times-out.
|
||||
#
|
||||
# Basically: this function doesn't do anything except wait to be
|
||||
# terminated.
|
||||
#
|
||||
# Vault-manager in MOUNT_HELPER has PVC mounted, allowing the real
|
||||
# vault-manager to read secrets from cluster_keys.json
|
||||
function mountHelper {
|
||||
local count
|
||||
|
||||
# omit this function if this pod is not the mount helper
|
||||
if [ -z "$MANAGER_MODE" -o "$MANAGER_MODE" != "MOUNT_HELPER" ]; then
|
||||
log $INFO "Mode is VAULT_MANAGER"
|
||||
return
|
||||
fi
|
||||
|
||||
# When vault-manager is running in this mode, it should be
|
||||
# deleted by vault-manager running in the default mode, which
|
||||
# is using this pod to read secrets from mounted PVC
|
||||
log $INFO "Mode is $MANAGER_MODE"
|
||||
|
||||
# start with some debug/error logs
|
||||
if [ -f "$PVC_DIR/cluster_keys.json" ]; then
|
||||
log $DEBUG "Successfully mounted secrets file"
|
||||
else
|
||||
log $WARNING "Secrets file not found"
|
||||
fi
|
||||
|
||||
# sleep for MOUNT_HELPER_MAX_TIME, expecting SIGTERM signal
|
||||
log $INFO "Waiting for termination request via SIGTERM"
|
||||
count=0
|
||||
while [ "$count" -lt "$MOUNT_HELPER_MAX_TIME" ]; do
|
||||
exit_on_trap
|
||||
count=$((count+1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Normally should exit by exit_on_trap, but here we timeout
|
||||
# waiting for the real vault-manager to delete this job/pod.
|
||||
log $INFO "Exiting without receiving SIGTERM request"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Check if a secret exists
|
||||
#
|
||||
# Returns the normal linux success=0, failure!=0
|
||||
# Prints the name of the secret
|
||||
function secretExists {
|
||||
local name="$1"
|
||||
kubectl get secrets -n vault "$name" \
|
||||
-o jsonpath='{.metadata.name}' 2>/dev/null \
|
||||
| grep "$name"
|
||||
}
|
||||
|
||||
# Check if the PVC resource exists
|
||||
#
|
||||
# Returns the normal linux success=0, failure!=0
|
||||
# Prints the name of the PVC resource
|
||||
function pvcExists {
|
||||
local text
|
||||
local jqscript
|
||||
|
||||
jqscript='.items
|
||||
| map(select(.metadata.name | test("^manager-pvc")))
|
||||
| .[0].metadata.name'
|
||||
|
||||
# using jq since kubernetes does not support regex
|
||||
# the grep makes sure the result contains the 'manager-pvc'
|
||||
# string (as opposed to 'null' for example)
|
||||
text="$(
|
||||
kubectl get persistentvolumeclaims -n vault -o json \
|
||||
| jq -r "$jqscript" 2>/dev/null \
|
||||
| grep manager-pvc )"
|
||||
result=$?
|
||||
|
||||
if [ -n "$text" ]; then
|
||||
echo "$text"
|
||||
fi
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
# Check if the PVC is mounted to any pod in vault namespace
|
||||
#
|
||||
# Returns the normal linux success=0, failure!=0
|
||||
# Prints the name of the PVC resource
|
||||
function testPVCMount {
|
||||
local result
|
||||
local cspec
|
||||
local vspec
|
||||
|
||||
cspec=".items[*].spec.containers[*]"
|
||||
vspec="volumeMounts[?(@.name=='manager-pvc')].name"
|
||||
|
||||
# this kubectl query returns zero whether manager-pvc is
|
||||
# found or not
|
||||
# result variable is either empty or 'manager-pvc'
|
||||
result="$( kubectl get pods -n vault \
|
||||
-o jsonpath="{${cspec}.${vspec}}" )"
|
||||
|
||||
if [ -n "$result" ]; then
|
||||
return 0
|
||||
fi
|
||||
return 1 # assertion 'fails'
|
||||
}
|
||||
|
||||
# This function prints a DEBUG log of kubectl delete
|
||||
function deleteMountHelper {
|
||||
local text
|
||||
local result
|
||||
|
||||
log $DEBUG "Waiting for delete of mount-helper job"
|
||||
text="$( kubectl delete --ignore-not-found=true --wait=true \
|
||||
-f /opt/yaml/pvc-attach.yaml 2>&1 )"
|
||||
result=$?
|
||||
log $DEBUG "Output of deleting mount-helper: [$text]"
|
||||
return $result
|
||||
}
|
||||
|
||||
# Run shred on the file content of PVC
|
||||
#
|
||||
# All files a shredded, and the result is an error if
|
||||
# - command return code is non-zero
|
||||
# - file comparison shows unchanged file(s)
|
||||
#
|
||||
# A warning is issued if shred/kubectl command has any stdout or
|
||||
# stderr
|
||||
#
|
||||
# Returns the normal linux success=0, failure!=0
|
||||
function securelyWipePVC {
|
||||
local helper="$1"
|
||||
|
||||
if [ -z "$helper" ]; then
|
||||
log $ERROR "No pod specified for shredding"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# get profile of the files before shredding
|
||||
kubectl exec -n vault "$helper" -- \
|
||||
bash -c 'find /mnt/data -type f \
|
||||
| sort | xargs wc | head -n-1' \
|
||||
>/tmp/shred_before.txt 2>&1
|
||||
log $DEBUG "Original files: [$( cat /tmp/shred_before.txt )]"
|
||||
|
||||
# run the shred command
|
||||
#
|
||||
# Shred all the files in mounted /mnt/data/
|
||||
#
|
||||
# The shred by default has three randomized passes, and with -z
|
||||
# option will finalize with zeros. -f prompts shred to work
|
||||
# around any unexpected file permissions
|
||||
text="$( kubectl exec -n vault "$helper" -- \
|
||||
bash -c '\
|
||||
result=0; \
|
||||
while read fname; do \
|
||||
shred -f -z "$fname"; \
|
||||
[ $? -ne 0 ] && result=1; \
|
||||
done <<<"$(find /mnt/data -type f )"; \
|
||||
exit $result' 2>&1 )"
|
||||
result=$?
|
||||
|
||||
# get profile of the files after shredding
|
||||
kubectl exec -n vault "$helper" -- \
|
||||
bash -c 'find /mnt/data -type f \
|
||||
| sort | xargs wc | head -n-1' \
|
||||
>/tmp/shred_after.txt 2>&1
|
||||
log $DEBUG "Shredded files: [$( cat /tmp/shred_after.txt )]"
|
||||
|
||||
# compare the profiles for error reporting
|
||||
#
|
||||
# If the file lists, pushed through wc, have files with the same
|
||||
# character, word, and line counts then report an error: a file
|
||||
# has not been shred
|
||||
#
|
||||
# Ignore files that were empty
|
||||
difftext="$( diff -wuU100000 /tmp/shred_before.txt \
|
||||
/tmp/shred_after.txt )"
|
||||
unchanged="$( echo "$difftext" | grep "^ " \
|
||||
| grep -v "^\([ ]\{1,\}0\)\{3\} /" )"
|
||||
|
||||
# Report the errors/success
|
||||
if [ "$result" -ne 0 ]; then
|
||||
log $ERROR "Error on shred: [$text]"
|
||||
if [ -n "$unchanged" ]; then
|
||||
log $ERROR "Unchanged: [$unchanged]"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
if [ -n "$text" ]; then
|
||||
log $WARNING "Output of shred is not empty: [$text]"
|
||||
fi
|
||||
if [ -n "$unchanged" ]; then
|
||||
log $ERROR "Shred did not shred some files"
|
||||
log $ERROR "Unchanged: [$unchanged]"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log $INFO "Shredding of PVC data verified"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Delete the PVC resource
|
||||
#
|
||||
# The delete will succeed even if attached to a pod, such as a
|
||||
# terminating vault-manager or mount-helper - the PVC remains
|
||||
# in terminating status until the pod is also terminated.
|
||||
function deletePVC {
|
||||
local text
|
||||
local name
|
||||
|
||||
name="$( pvcExists )"
|
||||
if [ $? -eq 0 ] && [[ "$name" =~ ^manager-pvc ]]; then
|
||||
text="$( kubectl delete persistentvolumeclaims -n vault \
|
||||
"$name" 2>&1 )"
|
||||
if [ $? -ne 0 ]; then
|
||||
log $ERROR "Error deleting PVC: [$text]"
|
||||
else
|
||||
log $INFO "$text"
|
||||
fi
|
||||
else
|
||||
log $WARNING "Request to delete PVC but PVC not found"
|
||||
fi
|
||||
}
|
||||
|
||||
# Delete the bootstrap secret
|
||||
function deleteBootstrap {
|
||||
local text
|
||||
text="$( kubectl delete secrets -n vault \
|
||||
cluster-key-bootstrap 2>&1 )"
|
||||
if [ $? -ne 0 ]; then
|
||||
log $ERROR "Error deleting bootstrap secret: [$text]"
|
||||
else
|
||||
log $INFO "$text"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run a job/pod, to mount the PVC resource, and retrieve the secrets
|
||||
# from PVC.
|
||||
#
|
||||
# See also the function mountHelper and the ConfigMap named:
|
||||
# {{ include "vault.name" . }}-mount-helper
|
||||
#
|
||||
# This function does not support overwriting an existing
|
||||
# cluster-key-* secret, but it does support validating those secrets
|
||||
# if they exist
|
||||
function convertPVC {
|
||||
local output
|
||||
local pod
|
||||
local count
|
||||
local text
|
||||
local PVCtext
|
||||
local result
|
||||
|
||||
if testPVCMount; then
|
||||
log $ERROR "Cannot mount PVC already mounted"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# run the pod
|
||||
output="$( kubectl apply -f /opt/yaml/pvc-attach.yaml 2>&1 )"
|
||||
if [ $? -ne 0 ]; then
|
||||
log $ERROR "Failed to apply mount-helper"
|
||||
log $DEBUG "Output: [$output]"
|
||||
deleteMountHelper
|
||||
return 1
|
||||
fi
|
||||
|
||||
# wait for pod
|
||||
pod=''
|
||||
count=0
|
||||
log $INFO "Waiting for mount-helper pod to run"
|
||||
while [ -z "$pod" -a "$count" -le "$MAX_POD_RUN_TRIES" ]; do
|
||||
count=$((count+1))
|
||||
text="$( kubectl get pods -n vault | grep "mount-helper" )"
|
||||
pod="$( echo "$text" | grep "Running" | awk '{print $1}' )"
|
||||
if [ -z "$pod" ]; then
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$pod" ]; then
|
||||
log $ERROR "Failed to run mount-helper pod"
|
||||
log $DEBUG "Pod state: [$( echo $text )]"
|
||||
deleteMountHelper
|
||||
return 1
|
||||
fi
|
||||
|
||||
# get the pvc data
|
||||
PVCtext="$( kubectl exec -n vault "$pod" \
|
||||
-- cat /mnt/data/cluster_keys.json )"
|
||||
if [ $? -ne 0 -o -z "$PVCtext" ]; then
|
||||
log $ERROR "Failed to read cluster_keys.json"
|
||||
deleteMountHelper
|
||||
return 1
|
||||
fi
|
||||
log $INFO "Data retrieved from PVC"
|
||||
|
||||
# if the Root secret is pre-existing, compare the existing
|
||||
# shard secrets and root secret before deleting the PVC
|
||||
kubectl get secrets -n vault cluster-key-root >/dev/null 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
log $INFO "Cluster secrets exist:" \
|
||||
"validating"
|
||||
else
|
||||
# create a secret from the data
|
||||
storeVaultInitSecrets "$PVCtext"
|
||||
fi
|
||||
|
||||
# verify the data stored versus text from PVC
|
||||
echo "$PVCtext" | validateSecrets
|
||||
result=$?
|
||||
if [ "$result" -eq 0 ]; then
|
||||
securelyWipePVC "$pod"
|
||||
# omit deleting the PVC for manual analysis and shred
|
||||
# when the wipe fails
|
||||
if [ $? -eq 0 ]; then
|
||||
deletePVC
|
||||
fi
|
||||
fi
|
||||
|
||||
# clean up but do not care about the result
|
||||
deleteMountHelper
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
function convertBootstrapSecrets {
|
||||
local text
|
||||
local count
|
||||
|
||||
text="$( get_secret cluster-key-bootstrap )"
|
||||
storeVaultInitSecrets "$text"
|
||||
|
||||
# verify the split secrets versus the bootstrap text
|
||||
echo "$text" | validateSecrets
|
||||
if [ $? -ne 0 ]; then
|
||||
# an error is already printed
|
||||
return 1
|
||||
fi
|
||||
|
||||
deleteBootstrap
|
||||
|
||||
# Also validate and delete the PVC resource
|
||||
# This procedure depends on waiting for the old version
|
||||
# of vault-manager pod to exit
|
||||
count="$TERMINATE_TRIES_MAX"
|
||||
log $INFO "Waiting for vault-manager pod to exit"
|
||||
while testPVCMount && [ "$count" -gt 0 ]; do
|
||||
sleep "$TERMINATE_TRIES_SLEEP"
|
||||
count=$((count-1))
|
||||
done
|
||||
|
||||
convertPVC
|
||||
}
|
||||
|
||||
function runConversion {
|
||||
if [ -n "$K8S_SECRETS_PREEXIST" ]; then
|
||||
log $INFO "Cluster secrets exist"
|
||||
return
|
||||
elif [ -n "$BOOTSTRAP_PREEXISTS" ]; then
|
||||
# this is the normal application update procedure; the
|
||||
# lifecycle code retrieved the secrets from previous version
|
||||
# of the application.
|
||||
log $INFO "Using secrets provided in $BOOTSTRAP_PREEXISTS"
|
||||
convertBootstrapSecrets
|
||||
return
|
||||
elif [ -z "$PVC_PREEXISTS" ]; then
|
||||
log $INFO "No pre-existing secrets exist"
|
||||
return
|
||||
fi
|
||||
|
||||
# Finally, read the pre-existing PVC. This occurs if the
|
||||
# application updates outside of application-update. For
|
||||
# example if the old application is removed and deleted, and the
|
||||
# new application is uploaded and applied.
|
||||
convertPVC
|
||||
}
|
||||
|
||||
|
||||
#
|
||||
# LOGIC
|
||||
#
|
||||
exit_on_trap 1
|
||||
|
||||
# check if this pod is helping to convert storage from pvc to k8s
|
||||
# secrets
|
||||
mountHelper
|
||||
exit_on_trap 15
|
||||
|
||||
# check if there are existing key shard secrets, boot strap secret,
|
||||
# or pre-existing resource
|
||||
K8S_SECRETS_PREEXIST="$( secretExists cluster-key-root )"
|
||||
exit_on_trap 16
|
||||
BOOTSTRAP_PREEXISTS="$( secretExists cluster-key-bootstrap )"
|
||||
exit_on_trap 17
|
||||
PVC_PREEXISTS="$( pvcExists )"
|
||||
exit_on_trap 18
|
||||
|
||||
runConversion
|
||||
exit_on_trap 19
|
||||
|
||||
# Waiting for at least one vault server, to check initialization
|
||||
waitForPods 1
|
||||
exit_on_trap 2
|
||||
@ -575,6 +1024,8 @@ data:
|
||||
sleep "$UNSEAL_RATE"
|
||||
exit_on_trap 8
|
||||
done
|
||||
else
|
||||
log $INFO "Vault is initialized"
|
||||
fi
|
||||
|
||||
exit_on_trap 9
|
||||
@ -629,6 +1080,71 @@ metadata:
|
||||
name: vault-init-unseal-2
|
||||
namespace: {{ .Release.Namespace }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
managedFields:
|
||||
- apiVersion: v1
|
||||
fieldsType: FieldsV1
|
||||
fieldsV1:
|
||||
f:data:
|
||||
.: {}
|
||||
f:pvc-attach.yaml: {}
|
||||
manager: {{ include "vault.name" . }}-mount-helper
|
||||
name: {{ include "vault.name" . }}-mount-helper
|
||||
namespace: {{ .Release.Namespace }}
|
||||
data:
|
||||
pvc-attach.yaml: |
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ template "vault.fullname" . }}-mount-helper
|
||||
namespace: vault
|
||||
spec:
|
||||
activeDeadlineSeconds: 600
|
||||
completions: 1
|
||||
parallelism: 1
|
||||
ttlSecondsAfterFinished: 0
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: "{{ template "vault.fullname" . }}-vault-manager"
|
||||
{{- if .Values.global.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.global.imagePullSecrets | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.manager.tolerations }}
|
||||
tolerations:
|
||||
{{- tpl .Values.manager.tolerations . | nindent 12 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: mount
|
||||
image: "{{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }}"
|
||||
imagePullPolicy: "{{ .Values.injector.image.pullPolicy }}"
|
||||
args:
|
||||
- bash
|
||||
- /opt/script/init.sh
|
||||
env:
|
||||
- name: MANAGER_MODE
|
||||
value: MOUNT_HELPER
|
||||
- name: PVC_DIR
|
||||
value: /mnt/data
|
||||
volumeMounts:
|
||||
- name: mount-helper
|
||||
mountPath: /opt/script
|
||||
readOnly: true
|
||||
- name: manager-pvc
|
||||
mountPath: /mnt/data
|
||||
readOnly: false
|
||||
volumes:
|
||||
- name: mount-helper
|
||||
configMap:
|
||||
name: vault-init-unseal-2
|
||||
- name: manager-pvc
|
||||
persistentVolumeClaim:
|
||||
claimName: manager-pvc-sva-vault-manager-0
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
@ -638,9 +1154,18 @@ rules:
|
||||
- apiGroups: [""] # "" indicates the core API group
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "watch", "list"]
|
||||
- apiGroups: [""] # "" indicates the core API group
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""] # "" indicates the core API group
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "create"]
|
||||
verbs: ["get", "create", "delete"]
|
||||
- apiGroups: ["batch"]
|
||||
resources: ["jobs"]
|
||||
verbs: ["get", "create", "delete"]
|
||||
- apiGroups: [""] # "" indicates the core API group
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["list", "delete"]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
@ -718,6 +1243,9 @@ spec:
|
||||
- name: vault-init-unseal-2
|
||||
mountPath: /opt/script
|
||||
readOnly: false
|
||||
- name: mount-helper-yaml
|
||||
mountPath: /opt/yaml
|
||||
readOnly: true
|
||||
- name: vault-ca
|
||||
mountPath: /mnt/data/ca
|
||||
readOnly: true
|
||||
@ -725,6 +1253,9 @@ spec:
|
||||
- name: vault-init-unseal-2
|
||||
configMap:
|
||||
name: vault-init-unseal-2
|
||||
- name: mount-helper-yaml
|
||||
configMap:
|
||||
name: {{ include "vault.name" . }}-mount-helper
|
||||
- name: vault-ca
|
||||
secret:
|
||||
secretName: vault-ca
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user