diff --git a/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/common/constants.py b/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/common/constants.py index a63bca1..5124470 100644 --- a/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/common/constants.py +++ b/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/common/constants.py @@ -17,3 +17,5 @@ HELM_VAULT_MANAGER_POD = 'manager' HELM_VAULT_INJECTOR_POD = 'injector' HELM_CHART_COMPONENT_LABEL = 'app.starlingx.io/component' + +KEYSHARDS = 5 diff --git a/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/lifecycle/__init__.py b/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/lifecycle/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/lifecycle/lifecycle_vault.py b/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/lifecycle/lifecycle_vault.py new file mode 100644 index 0000000..2557eaf --- /dev/null +++ b/python3-k8sapp-vault/k8sapp_vault/k8sapp_vault/lifecycle/lifecycle_vault.py @@ -0,0 +1,278 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# All Rights Reserved. +# + +""" System inventory App lifecycle operator.""" + +from base64 import b64encode +import json +from k8sapp_vault.common import constants as app_constants +from oslo_log import log as logging +from sysinv.common import constants +from sysinv.common import kubernetes +from sysinv.common import utils as cutils +from sysinv.helm import lifecycle_base as base +import time + +LOG = logging.getLogger(__name__) + +CONF = kubernetes.KUBERNETES_ADMIN_CONF +NS = app_constants.HELM_CHART_NS_VAULT + +# wait parameters for kubernetes secret creation +WAIT_INTERVAL = 1 # seconds +WAIT_COUNT = 10 + + +class VaultAppLifecycleOperator(base.AppLifecycleOperator): + """Lifecycle operator for vault application""" + + def app_lifecycle_actions(self, context, conductor_obj, app_op, app, hook_info): + """Perform lifecycle actions for an operation + + :param context: request context, can be None + :param conductor_obj: conductor object, can be None + :param app_op: AppOperator object + :param app: AppOperator.Application object + :param hook_info: LifecycleHookInfo object + + """ + if (hook_info.lifecycle_type == constants.APP_LIFECYCLE_TYPE_RESOURCE + and hook_info.operation == constants.APP_APPLY_OP + and hook_info.relative_timing == constants.APP_LIFECYCLE_TIMING_PRE): + try: + self.read_pvc_secret(app_op._kube._get_kubernetesclient_core()) + except Exception: # nosec # pylint: disable=broad-exception-caught + # omit printing all exceptions in case any may + # contain secret data + pass + + super().app_lifecycle_actions(context, conductor_obj, app_op, + app, hook_info) + + def validate_document(self, jdoc): + """Check whether the keyshards json is expected""" + error = False + + if not isinstance(jdoc, dict): + LOG.error("document is not dict type") + return False + if not len(jdoc) == 3: + LOG.error("document is not length 3") + error = True + + if 'keys' not in jdoc.keys(): + LOG.error("keys not in document") + error = True + elif not isinstance(jdoc['keys'], list): + LOG.error("keys is not list type") + error = True + elif not len(jdoc['keys']) == app_constants.KEYSHARDS: + LOG.error("len(keys) not expected") + error = True + + if 'keys_base64' not in jdoc.keys(): + LOG.error("keys_base64 not in document") + error = True + elif not isinstance(jdoc['keys_base64'], list): + LOG.error("keys_base64 is not list type") + error = True + elif not len(jdoc['keys_base64']) == app_constants.KEYSHARDS: + LOG.error("len(keys_base64) not expected") + error = True + + if 'root_token' not in jdoc.keys(): + LOG.error("root_token not in document") + error = True + elif not isinstance(jdoc['root_token'], str): + LOG.error("root_token not str type") + error = True + + return not error + + def ns_exists(self): + """check if vault is listed in namespaces""" + jsonpath = '{.items[*].metadata.name}' + cmd = ['kubectl', '--kubeconfig', CONF, + 'get', 'ns', '-o', 'jsonpath=' + jsonpath] + stdout, stderr = cutils.trycmd(*cmd) + if not stdout: + LOG.info('Failed to get namespaces [%s]', stderr) + return False + if NS not in stdout.split(): + LOG.info('No vault namespace') + return False + return True + + def get_pod_list(self): + """Get all pods in vault ns""" + + if not self.ns_exists(): + return [] + + jsonpath = '{.items[*].metadata.name}' + cmd = ['kubectl', '--kubeconfig', CONF, + 'get', 'pods', '-n', NS, + '-o', 'jsonpath=' + jsonpath] + stdout, stderr = cutils.trycmd(*cmd) + if not stdout: + LOG.info('No pods in vault namespace: [%s]', stderr) + return [] + + return stdout.split() + + def get_manager_pods(self): + """Get all pods named sva-vault-manager""" + + managers = [] + pods = self.get_pod_list() + for pod in pods: + if pod.startswith('sva-vault-manager'): + managers.append(pod) + if not managers: + LOG.info('failed to get vault-manager pod') + return [] + return managers + + def get_manager_pod(self): + """Return pod name if it has PVC mounted""" + pods = self.get_manager_pods() + + # assert that the vault-manager pod has PVC mounted + managerpod = '' + cspec = ".spec.containers[?(@.name=='manager')]" + vspec = "volumeMounts[?(@.name=='manager-pvc')].name" + jsonpath = "{%s.%s}" % (cspec, vspec) + for pod in pods: + cmd = ['kubectl', '--kubeconfig', CONF, + 'get', 'pods', '-n', NS, pod, '-o', + 'jsonpath=' + jsonpath] + stdout, stderr = cutils.trycmd(*cmd) + if stderr or not stdout: + LOG.debug('vault-manager pod without PVC mounted' + '[%s]', stderr) + continue + if managerpod: + LOG.info('More than one vault-manager pod with PVC mounted' + '[%s] and [%s]', managerpod, pod) + managerpod = pod + LOG.info('vault-manager pod with PVC mounted:' + '[%s]', managerpod) + return managerpod + + def get_key_shards(self, podname): + """Read the key shards from vault-manager pod""" + + cmd = ['kubectl', 'exec', '-n', NS, podname, + '--kubeconfig', CONF, + '--', 'cat', '/mnt/data/cluster_keys.json'] + stdout, stderr = cutils.trycmd(*cmd) + if stderr or not stdout: + LOG.info('cluster keys missing from PVC storage') + return '' + return stdout.strip() + + def create_secret(self, client, shards): + """create a secret from shards text""" + metadata = {'name': 'cluster-key-bootstrap'} + data = {'strdata': shards} + body = {'apiVersion': 'v1', + 'metadata': metadata, + 'data': data, + 'kind': 'Secret'} + try: + api_response = client.create_namespaced_secret(NS, body) + except kubernetes.client.exceptions.ApiException: + # omitting printing the exception text in case it may + # contain the secrets content + LOG.error('Failed to create bootstrap secret ' + '(ApiException)') + return False + + # verify that the api response contains the secret + if ('data' in dir(api_response) + and 'strdata' in api_response.data + and api_response.data['strdata'] == shards): + LOG.info('API response includes correct data') + else: + LOG.error('Failed to verify kubernetes api response') + + # Ignore the above verification and continue + return True + + def read_pvc_secret(self, client): + """Retrieve key shards from a running vault-manager pod + + The key shards are stored into k8s secrete + 'cluster-key-bootstrap', to be consumed by the new vault-manager + pod. The vault-manager will also delete the PVC resource after + successful validations. + + Do nothing if: + - no vault-manager pod is running with PVC attached (i.e.: no + namespace, no pod. no vault-manager or no PVC attached) + - PVC does not contain the expected key shards file + - key shards data is not in an expected format (data structure, + number of key shards + + Print only soft errors if the validation of stored k8s secret + is not successful. + """ + + podname = self.get_manager_pod() + if not podname: + LOG.info('No vault-manager with PVC mounted') + return + + keyshards = self.get_key_shards(podname) + if not keyshards: + # an error is printed + return + + # using encode()/decode() because b64encode requires bytes, but + # kubernetes api requires str + b64_keyshards = b64encode(keyshards.encode()).decode() + + # assert that it's a json document with expected keys + try: + document = json.loads(keyshards) + except json.decoder.JSONDecodeError: + LOG.error("Failed to parse json document") + return + + if self.validate_document(document): + LOG.info("Successfully retrieved %s key shards", + len(document['keys'])) + else: + LOG.error('The data appears invalid') + return + + if not self.create_secret(client, b64_keyshards): + # an error is already printed + return + + # read the secret back + LOG.info('Wait for the secret to be created') + count = WAIT_COUNT + while count > 0: + # read the secret back + jsonpath = "{.data.strdata}" + cmd = ['kubectl', '--kubeconfig', CONF, + 'get', 'secrets', '-n', NS, 'cluster-key-bootstrap', + '-o', 'jsonpath=' + jsonpath] + stdout, stderr = cutils.trycmd(*cmd) + if stdout and b64_keyshards == stdout: + break + LOG.debug('Result kubectl get secret ' + 'cluster-key-bootstrap: [%s]', stderr) + count -= 1 + time.sleep(WAIT_INTERVAL) + + if b64_keyshards == stdout: + LOG.info('Validation of stored key shards successful') + else: + LOG.error('Validation of stored key shards failed') diff --git a/python3-k8sapp-vault/k8sapp_vault/setup.cfg b/python3-k8sapp-vault/k8sapp_vault/setup.cfg index 9bb2267..4cfe627 100644 --- a/python3-k8sapp-vault/k8sapp_vault/setup.cfg +++ b/python3-k8sapp-vault/k8sapp_vault/setup.cfg @@ -34,5 +34,9 @@ systemconfig.helm_applications = systemconfig.helm_plugins.vault = 001_vault = k8sapp_vault.helm.vault:VaultHelm + +systemconfig.app_lifecycle = + vault = k8sapp_vault.lifecycle.lifecycle_vault:VaultAppLifecycleOperator + [bdist_wheel] universal = 1 diff --git a/vault-helm/vault-helm/helm-charts/vault-init.yaml b/vault-helm/vault-helm/helm-charts/vault-init.yaml index 0876703..2a4b47d 100644 --- a/vault-helm/vault-helm/helm-charts/vault-init.yaml +++ b/vault-helm/vault-helm/helm-charts/vault-init.yaml @@ -31,6 +31,22 @@ data: # reserve trap '0' for disabling a debugging trap request DEBUGGING_TRAP=0 + # Maximum sleep seconds for mount-helper before exiting + MOUNT_HELPER_MAX_TIME=60 + + # Maximum seconds to wait for mount-helper pod to start + MAX_POD_RUN_TRIES=10 + + # Maximum seconds to wait for vault-manager pod to exit + # Vault-manager is not responding to SIGTERM, so will take 30 + # seconds + TERMINATE_TRIES_MAX=6 + TERMINATE_TRIES_SLEEP=5 + + # Vault key share configuration + KEY_SECRET_SHARES=5 + KEY_REQUIRED_THRESHOLD=3 + # Records for seal status state machine: PODREC_F="$WORKDIR/previous_pods_status.txt" PODREC_TMP_F="$WORKDIR/new_pods_status.txt" @@ -70,9 +86,6 @@ data: LOG_LEVEL={{ .Values.manager.log.defaultLogLevel }} LOG_OVERRIDE_FILE="$WORKDIR/log_level" - # Number of key shards - KEY_SECRET_SHARES=5 - # FUNCTIONS # Convert log level to text for log message @@ -217,7 +230,10 @@ data: rm -f "$WORKDIR"/s1 "$WORKDIR"/s2 } - # validation function for splitShard + # Check the structure of json data and confirm equivalence of + # the stdin with stored secrets + # + # Returns the normal linux success=0, failure!=0 function validateSecrets { local index local text @@ -234,6 +250,8 @@ data: keys=$( echo "$text" | jq '.keys' ) keys_base64=$( echo "$text" | jq '.keys_base64' ) root_token=$( echo "$text" | jq -r '.root_token' ) + # response is 'null' if the dict key is missing + # response is empty (-z) is the source document is empty if [ -z "$keys" -o "$keys" == "null" \ -o -z "$keys_base64" -o "$keys_base64" == "null" \ -o -z "$root_token" -o "$root_token" == "null" ]; then @@ -242,14 +260,22 @@ data: fi count=$( echo "$keys" | jq '. | length' ) + if [ $? -ne 0 ]; then + log $ERROR "jq did not parse keys length" + return 1 + fi if [ -z "$count" ] || [ "$count" -ne "$KEY_SECRET_SHARES" ]; then - log $ERROR "Incorrect array length for keys: " \ + log $ERROR "Incorrect array length for keys:" \ "$count instead of $KEY_SECRET_SHARES" return 1 fi count=$( echo "$keys_base64" | jq '. | length' ) + if [ $? -ne 0 ]; then + log $ERROR "jq did not parse keys_base64 length" + return 1 + fi if [ -z "$count" ] || [ "$count" -ne "$KEY_SECRET_SHARES" ]; then - log $ERROR "Incorrect array length for keys_base64: " \ + log $ERROR "Incorrect array length for keys_base64:" \ "$count instead of $KEY_SECRET_SHARES" return 1 fi @@ -267,6 +293,9 @@ data: root_saved="$( get_secret cluster-key-root )" saved=$( echo "$saved" | jq \ -c '{keys: .keys, keys_base64: .keys_base64, root_token: "'$root_saved'"}' ) + + # finally ensure that the saved secrets are the same as the + # supplied text shaA=$( echo "$text" | sha256sum ) shaB=$( echo "$saved" | sha256sum ) if [ "$shaA" != "$shaB" ]; then @@ -274,7 +303,7 @@ data: return 1 fi - log $INFO "Verified creation of secret" + log $INFO "Verified stored secrets are the same as supplied data" return 0 } @@ -323,25 +352,44 @@ data: done } + # Takes the json document output from vault initialization + # and stores it into secrets for key shards and the root token + # + # This only works if the secrets are not pre-existing. An error + # is printed by set_secrets. + function storeVaultInitSecrets { + local secrets="$1" + local index + local split_json + + for index in $(seq 0 $((KEY_SECRET_SHARES - 1 ))); do + split_json=$( echo -n "$secrets" | splitShard "$index" ) + set_secret "cluster-key-$index" /dev/stdin <<< "$split_json" + done + + split_json=$( echo "$secrets" | jq -r '.root_token' ) + set_secret "cluster-key-root" /dev/stdin <<< "$split_json" + } + # Initializes the first vault pod, only needs to be performed once # after deploying the helm chart # Stores the root token and master key shards in k8s secrets function initVault { + local V0 local keys - local index - local split_json - local secret_json local key_error + local shares + local threshold V0=$(awk 'NR==1{print $2}' $WORKDIR/pods.txt) log $INFO "Initializing $V0" - secret_json="{\"secret_shares\": $KEY_SECRET_SHARES, \"secret_threshold\": 3}" - + shares='"secret_shares": '$KEY_SECRET_SHARES + threshold='"secret_threshold": '$KEY_REQUIRED_THRESHOLD keys=$( curl -s \ --cacert $CERT \ --request POST \ - --data "$secret_json" \ + --data "{$shares, $threshold}" \ https://$V0.$DOMAIN:8200/v1/sys/init ) key_error=$(echo -n "$keys"| jq -r '.errors[]?') @@ -349,15 +397,9 @@ data: log $ERROR "vault init request failed: $key_error" fi - for index in $(seq 0 $((KEY_SECRET_SHARES - 1 ))); do - split_json=$( echo -n "$keys" | splitShard "$index" ) - set_secret "cluster-key-$index" /dev/stdin <<< "$split_json" - done + storeVaultInitSecrets "$keys" - split_json=$( echo "$keys" | jq -r '.root_token' ) - set_secret "cluster-key-root" /dev/stdin <<< "$split_json" - - # validation if the split and store secrets is identical to the original pulled. + # check if the secrets match vault's REST API response echo "$keys" | validateSecrets } @@ -506,9 +548,17 @@ data: function set_secret { local secret="$1" local contentf="$2" + local output + local result - kubectl create secret generic -n "$VAULT_NS" "$secret" \ - "--from-file=strdata=$contentf" + output="$( kubectl create secret generic -n "$VAULT_NS" \ + "$secret" "--from-file=strdata=$contentf" 2>&1 )" + result=$? + if [ "$result" -ne 0 ]; then + log $ERROR "Failed to create secret $secret" + log $DEBUG "Output: [$output]" + fi + return $result } function get_secret { @@ -519,11 +569,410 @@ data: | base64 -d } + # When vault-manager is run in "MOUNT_HELPER" mode, this function + # will not return. Instead the function will exit_on_trap or exit + # when it times-out. + # + # Basically: this function doesn't do anything except wait to be + # terminated. + # + # Vault-manager in MOUNT_HELPER has PVC mounted, allowing the real + # vault-manager to read secrets from cluster_keys.json + function mountHelper { + local count + + # omit this function if this pod is not the mount helper + if [ -z "$MANAGER_MODE" -o "$MANAGER_MODE" != "MOUNT_HELPER" ]; then + log $INFO "Mode is VAULT_MANAGER" + return + fi + + # When vault-manager is running in this mode, it should be + # deleted by vault-manager running in the default mode, which + # is using this pod to read secrets from mounted PVC + log $INFO "Mode is $MANAGER_MODE" + + # start with some debug/error logs + if [ -f "$PVC_DIR/cluster_keys.json" ]; then + log $DEBUG "Successfully mounted secrets file" + else + log $WARNING "Secrets file not found" + fi + + # sleep for MOUNT_HELPER_MAX_TIME, expecting SIGTERM signal + log $INFO "Waiting for termination request via SIGTERM" + count=0 + while [ "$count" -lt "$MOUNT_HELPER_MAX_TIME" ]; do + exit_on_trap + count=$((count+1)) + sleep 1 + done + + # Normally should exit by exit_on_trap, but here we timeout + # waiting for the real vault-manager to delete this job/pod. + log $INFO "Exiting without receiving SIGTERM request" + exit 0 + } + + # Check if a secret exists + # + # Returns the normal linux success=0, failure!=0 + # Prints the name of the secret + function secretExists { + local name="$1" + kubectl get secrets -n vault "$name" \ + -o jsonpath='{.metadata.name}' 2>/dev/null \ + | grep "$name" + } + + # Check if the PVC resource exists + # + # Returns the normal linux success=0, failure!=0 + # Prints the name of the PVC resource + function pvcExists { + local text + local jqscript + + jqscript='.items + | map(select(.metadata.name | test("^manager-pvc"))) + | .[0].metadata.name' + + # using jq since kubernetes does not support regex + # the grep makes sure the result contains the 'manager-pvc' + # string (as opposed to 'null' for example) + text="$( + kubectl get persistentvolumeclaims -n vault -o json \ + | jq -r "$jqscript" 2>/dev/null \ + | grep manager-pvc )" + result=$? + + if [ -n "$text" ]; then + echo "$text" + fi + + return $result + } + + # Check if the PVC is mounted to any pod in vault namespace + # + # Returns the normal linux success=0, failure!=0 + # Prints the name of the PVC resource + function testPVCMount { + local result + local cspec + local vspec + + cspec=".items[*].spec.containers[*]" + vspec="volumeMounts[?(@.name=='manager-pvc')].name" + + # this kubectl query returns zero whether manager-pvc is + # found or not + # result variable is either empty or 'manager-pvc' + result="$( kubectl get pods -n vault \ + -o jsonpath="{${cspec}.${vspec}}" )" + + if [ -n "$result" ]; then + return 0 + fi + return 1 # assertion 'fails' + } + + # This function prints a DEBUG log of kubectl delete + function deleteMountHelper { + local text + local result + + log $DEBUG "Waiting for delete of mount-helper job" + text="$( kubectl delete --ignore-not-found=true --wait=true \ + -f /opt/yaml/pvc-attach.yaml 2>&1 )" + result=$? + log $DEBUG "Output of deleting mount-helper: [$text]" + return $result + } + + # Run shred on the file content of PVC + # + # All files a shredded, and the result is an error if + # - command return code is non-zero + # - file comparison shows unchanged file(s) + # + # A warning is issued if shred/kubectl command has any stdout or + # stderr + # + # Returns the normal linux success=0, failure!=0 + function securelyWipePVC { + local helper="$1" + + if [ -z "$helper" ]; then + log $ERROR "No pod specified for shredding" + return 1 + fi + + # get profile of the files before shredding + kubectl exec -n vault "$helper" -- \ + bash -c 'find /mnt/data -type f \ + | sort | xargs wc | head -n-1' \ + >/tmp/shred_before.txt 2>&1 + log $DEBUG "Original files: [$( cat /tmp/shred_before.txt )]" + + # run the shred command + # + # Shred all the files in mounted /mnt/data/ + # + # The shred by default has three randomized passes, and with -z + # option will finalize with zeros. -f prompts shred to work + # around any unexpected file permissions + text="$( kubectl exec -n vault "$helper" -- \ + bash -c '\ + result=0; \ + while read fname; do \ + shred -f -z "$fname"; \ + [ $? -ne 0 ] && result=1; \ + done <<<"$(find /mnt/data -type f )"; \ + exit $result' 2>&1 )" + result=$? + + # get profile of the files after shredding + kubectl exec -n vault "$helper" -- \ + bash -c 'find /mnt/data -type f \ + | sort | xargs wc | head -n-1' \ + >/tmp/shred_after.txt 2>&1 + log $DEBUG "Shredded files: [$( cat /tmp/shred_after.txt )]" + + # compare the profiles for error reporting + # + # If the file lists, pushed through wc, have files with the same + # character, word, and line counts then report an error: a file + # has not been shred + # + # Ignore files that were empty + difftext="$( diff -wuU100000 /tmp/shred_before.txt \ + /tmp/shred_after.txt )" + unchanged="$( echo "$difftext" | grep "^ " \ + | grep -v "^\([ ]\{1,\}0\)\{3\} /" )" + + # Report the errors/success + if [ "$result" -ne 0 ]; then + log $ERROR "Error on shred: [$text]" + if [ -n "$unchanged" ]; then + log $ERROR "Unchanged: [$unchanged]" + fi + return 1 + fi + if [ -n "$text" ]; then + log $WARNING "Output of shred is not empty: [$text]" + fi + if [ -n "$unchanged" ]; then + log $ERROR "Shred did not shred some files" + log $ERROR "Unchanged: [$unchanged]" + return 1 + fi + + log $INFO "Shredding of PVC data verified" + + return 0 + } + + # Delete the PVC resource + # + # The delete will succeed even if attached to a pod, such as a + # terminating vault-manager or mount-helper - the PVC remains + # in terminating status until the pod is also terminated. + function deletePVC { + local text + local name + + name="$( pvcExists )" + if [ $? -eq 0 ] && [[ "$name" =~ ^manager-pvc ]]; then + text="$( kubectl delete persistentvolumeclaims -n vault \ + "$name" 2>&1 )" + if [ $? -ne 0 ]; then + log $ERROR "Error deleting PVC: [$text]" + else + log $INFO "$text" + fi + else + log $WARNING "Request to delete PVC but PVC not found" + fi + } + + # Delete the bootstrap secret + function deleteBootstrap { + local text + text="$( kubectl delete secrets -n vault \ + cluster-key-bootstrap 2>&1 )" + if [ $? -ne 0 ]; then + log $ERROR "Error deleting bootstrap secret: [$text]" + else + log $INFO "$text" + fi + } + + # Run a job/pod, to mount the PVC resource, and retrieve the secrets + # from PVC. + # + # See also the function mountHelper and the ConfigMap named: + # {{ include "vault.name" . }}-mount-helper + # + # This function does not support overwriting an existing + # cluster-key-* secret, but it does support validating those secrets + # if they exist + function convertPVC { + local output + local pod + local count + local text + local PVCtext + local result + + if testPVCMount; then + log $ERROR "Cannot mount PVC already mounted" + return 1 + fi + + # run the pod + output="$( kubectl apply -f /opt/yaml/pvc-attach.yaml 2>&1 )" + if [ $? -ne 0 ]; then + log $ERROR "Failed to apply mount-helper" + log $DEBUG "Output: [$output]" + deleteMountHelper + return 1 + fi + + # wait for pod + pod='' + count=0 + log $INFO "Waiting for mount-helper pod to run" + while [ -z "$pod" -a "$count" -le "$MAX_POD_RUN_TRIES" ]; do + count=$((count+1)) + text="$( kubectl get pods -n vault | grep "mount-helper" )" + pod="$( echo "$text" | grep "Running" | awk '{print $1}' )" + if [ -z "$pod" ]; then + sleep 1 + fi + done + + if [ -z "$pod" ]; then + log $ERROR "Failed to run mount-helper pod" + log $DEBUG "Pod state: [$( echo $text )]" + deleteMountHelper + return 1 + fi + + # get the pvc data + PVCtext="$( kubectl exec -n vault "$pod" \ + -- cat /mnt/data/cluster_keys.json )" + if [ $? -ne 0 -o -z "$PVCtext" ]; then + log $ERROR "Failed to read cluster_keys.json" + deleteMountHelper + return 1 + fi + log $INFO "Data retrieved from PVC" + + # if the Root secret is pre-existing, compare the existing + # shard secrets and root secret before deleting the PVC + kubectl get secrets -n vault cluster-key-root >/dev/null 2>&1 + if [ $? -eq 0 ]; then + log $INFO "Cluster secrets exist:" \ + "validating" + else + # create a secret from the data + storeVaultInitSecrets "$PVCtext" + fi + + # verify the data stored versus text from PVC + echo "$PVCtext" | validateSecrets + result=$? + if [ "$result" -eq 0 ]; then + securelyWipePVC "$pod" + # omit deleting the PVC for manual analysis and shred + # when the wipe fails + if [ $? -eq 0 ]; then + deletePVC + fi + fi + + # clean up but do not care about the result + deleteMountHelper + + return $result + } + + function convertBootstrapSecrets { + local text + local count + + text="$( get_secret cluster-key-bootstrap )" + storeVaultInitSecrets "$text" + + # verify the split secrets versus the bootstrap text + echo "$text" | validateSecrets + if [ $? -ne 0 ]; then + # an error is already printed + return 1 + fi + + deleteBootstrap + + # Also validate and delete the PVC resource + # This procedure depends on waiting for the old version + # of vault-manager pod to exit + count="$TERMINATE_TRIES_MAX" + log $INFO "Waiting for vault-manager pod to exit" + while testPVCMount && [ "$count" -gt 0 ]; do + sleep "$TERMINATE_TRIES_SLEEP" + count=$((count-1)) + done + + convertPVC + } + + function runConversion { + if [ -n "$K8S_SECRETS_PREEXIST" ]; then + log $INFO "Cluster secrets exist" + return + elif [ -n "$BOOTSTRAP_PREEXISTS" ]; then + # this is the normal application update procedure; the + # lifecycle code retrieved the secrets from previous version + # of the application. + log $INFO "Using secrets provided in $BOOTSTRAP_PREEXISTS" + convertBootstrapSecrets + return + elif [ -z "$PVC_PREEXISTS" ]; then + log $INFO "No pre-existing secrets exist" + return + fi + + # Finally, read the pre-existing PVC. This occurs if the + # application updates outside of application-update. For + # example if the old application is removed and deleted, and the + # new application is uploaded and applied. + convertPVC + } + + # # LOGIC # exit_on_trap 1 + # check if this pod is helping to convert storage from pvc to k8s + # secrets + mountHelper + exit_on_trap 15 + + # check if there are existing key shard secrets, boot strap secret, + # or pre-existing resource + K8S_SECRETS_PREEXIST="$( secretExists cluster-key-root )" + exit_on_trap 16 + BOOTSTRAP_PREEXISTS="$( secretExists cluster-key-bootstrap )" + exit_on_trap 17 + PVC_PREEXISTS="$( pvcExists )" + exit_on_trap 18 + + runConversion + exit_on_trap 19 + # Waiting for at least one vault server, to check initialization waitForPods 1 exit_on_trap 2 @@ -575,6 +1024,8 @@ data: sleep "$UNSEAL_RATE" exit_on_trap 8 done + else + log $INFO "Vault is initialized" fi exit_on_trap 9 @@ -629,6 +1080,71 @@ metadata: name: vault-init-unseal-2 namespace: {{ .Release.Namespace }} --- +apiVersion: v1 +kind: ConfigMap +metadata: + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:pvc-attach.yaml: {} + manager: {{ include "vault.name" . }}-mount-helper + name: {{ include "vault.name" . }}-mount-helper + namespace: {{ .Release.Namespace }} +data: + pvc-attach.yaml: | + --- + apiVersion: batch/v1 + kind: Job + metadata: + name: {{ template "vault.fullname" . }}-mount-helper + namespace: vault + spec: + activeDeadlineSeconds: 600 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 0 + template: + spec: + restartPolicy: Never + serviceAccountName: "{{ template "vault.fullname" . }}-vault-manager" + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.global.imagePullSecrets | nindent 12 }} + {{- end }} + {{- if .Values.manager.tolerations }} + tolerations: + {{- tpl .Values.manager.tolerations . | nindent 12 }} + {{- end }} + containers: + - name: mount + image: "{{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }}" + imagePullPolicy: "{{ .Values.injector.image.pullPolicy }}" + args: + - bash + - /opt/script/init.sh + env: + - name: MANAGER_MODE + value: MOUNT_HELPER + - name: PVC_DIR + value: /mnt/data + volumeMounts: + - name: mount-helper + mountPath: /opt/script + readOnly: true + - name: manager-pvc + mountPath: /mnt/data + readOnly: false + volumes: + - name: mount-helper + configMap: + name: vault-init-unseal-2 + - name: manager-pvc + persistentVolumeClaim: + claimName: manager-pvc-sva-vault-manager-0 +--- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -638,9 +1154,18 @@ rules: - apiGroups: [""] # "" indicates the core API group resources: ["pods"] verbs: ["get", "watch", "list"] +- apiGroups: [""] # "" indicates the core API group + resources: ["pods/exec"] + verbs: ["create"] - apiGroups: [""] # "" indicates the core API group resources: ["secrets"] - verbs: ["get", "create"] + verbs: ["get", "create", "delete"] +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "create", "delete"] +- apiGroups: [""] # "" indicates the core API group + resources: ["persistentvolumeclaims"] + verbs: ["list", "delete"] --- apiVersion: v1 kind: ServiceAccount @@ -718,6 +1243,9 @@ spec: - name: vault-init-unseal-2 mountPath: /opt/script readOnly: false + - name: mount-helper-yaml + mountPath: /opt/yaml + readOnly: true - name: vault-ca mountPath: /mnt/data/ca readOnly: true @@ -725,6 +1253,9 @@ spec: - name: vault-init-unseal-2 configMap: name: vault-init-unseal-2 + - name: mount-helper-yaml + configMap: + name: {{ include "vault.name" . }}-mount-helper - name: vault-ca secret: secretName: vault-ca