diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f51ee66 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.tox +venv/ \ No newline at end of file diff --git a/.zuul.yaml b/.zuul.yaml index 99f7a9b..8821e3b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,9 +1,71 @@ --- - project: + vars: + ensure_tox_version: '<4' check: jobs: - - noop + - openstack-tox-linters + - k8sapp-openbao-tox-pylint + - k8sapp-openbao-tox-flake8 + - k8sapp-openbao-tox-metadata gate: jobs: - - noop + - openstack-tox-linters + - k8sapp-openbao-tox-pylint + - k8sapp-openbao-tox-flake8 + - k8sapp-openbao-tox-metadata +- job: + name: k8sapp-openbao-tox-metadata + parent: tox + description: | + Run metadata test for k8sapp_openbao + nodeset: debian-bullseye + required-projects: + - starlingx/config + - starlingx/fault + - starlingx/update + - starlingx/utilities + - starlingx/root + vars: + tox_envlist: metadata + tox_extra_args: -c python3-k8sapp-openbao/k8sapp_openbao/tox.ini + tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt' + +- job: + name: k8sapp-openbao-tox-pylint + parent: tox + description: | + Run pylint test for k8sapp_openbao + nodeset: debian-bullseye + required-projects: + - starlingx/config + - starlingx/fault + - starlingx/update + - starlingx/utilities + - starlingx/root + files: + - python3-k8sapp-openbao/* + vars: + tox_envlist: pylint + tox_extra_args: -c python3-k8sapp-openbao/k8sapp_openbao/tox.ini + tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt' + +- job: + name: k8sapp-openbao-tox-flake8 + parent: tox + description: | + Run flake8 test for k8sapp_openbao + nodeset: debian-bullseye + required-projects: + - starlingx/config + - starlingx/fault + - starlingx/update + - starlingx/utilities + - starlingx/root + files: + - python3-k8sapp-openbao/* + vars: + tox_envlist: flake8 + tox_extra_args: -c python3-k8sapp-openbao/k8sapp_openbao/tox.ini + tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt' diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..f2f8e6d --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,16 @@ +If you would like to contribute to the development of OpenStack, +you must follow the steps in this page: + + https://docs.openstack.org/infra/manual/developers.html + +Once those steps have been completed, changes to OpenStack +should be submitted for review via the Gerrit tool, following +the workflow documented at: + + https://docs.openstack.org/infra/manual/developers.html#development-workflow + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed in Launchpad: + + https://bugs.launchpad.net/starlingx diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000..e2797ea --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,17 @@ +StarlingX App-Openbao Style Commandments +================================================================ + +- Step 1: Read the OpenStack style commandments + https://docs.openstack.org/hacking/latest/ +- Step 2: Read on + +App-Openbao Specific Commandments +--------------------------------------------------------- + +None so far + +Running tests +------------- +The approach to running tests is to simply run the command ``tox``. This will +create virtual environments, populate them with dependencies and run all of +the tests that OpenStack CI systems run. diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..3ffe69f --- /dev/null +++ b/bindep.txt @@ -0,0 +1,10 @@ +# This is a cross-platform list tracking distribution packages needed for install and tests; +# see https://docs.openstack.org/infra/bindep/ for additional information. + +libffi-dev [platform:dpkg] +libldap2-dev [platform:dpkg] +libxml2-dev [platform:dpkg] +libxslt1-dev [platform:dpkg] +libsasl2-dev [platform:dpkg] +libffi-devel [platform:rpm] +python3-all-dev [platform:dpkg] diff --git a/debian_build_layer.cfg b/debian_build_layer.cfg new file mode 100644 index 0000000..c581999 --- /dev/null +++ b/debian_build_layer.cfg @@ -0,0 +1 @@ +flock diff --git a/debian_iso_image.inc b/debian_iso_image.inc new file mode 100644 index 0000000..80d46cc --- /dev/null +++ b/debian_iso_image.inc @@ -0,0 +1,2 @@ +#stx-openbao-helm +stx-openbao-helm diff --git a/debian_pkg_dirs b/debian_pkg_dirs new file mode 100644 index 0000000..92bd538 --- /dev/null +++ b/debian_pkg_dirs @@ -0,0 +1,4 @@ +python3-k8sapp-openbao +stx-openbao-helm +helm-charts/upstream/openbao-helm +helm-charts/custom/openbao-manager-helm diff --git a/helm-charts/custom/openbao-manager-helm/debian/deb_folder/changelog b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/changelog new file mode 100644 index 0000000..b1e1ca0 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/changelog @@ -0,0 +1,5 @@ +openbao-manager-helm (1.0-1) unstable; urgency=medium + + * Initial release. + + -- Tae Park Thu, 10 Oct 2024 19:34:18 -0400 diff --git a/helm-charts/custom/openbao-manager-helm/debian/deb_folder/control b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/control new file mode 100644 index 0000000..1f87c60 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/control @@ -0,0 +1,16 @@ +Source: openbao-manager-helm +Section: libs +Priority: optional +Maintainer: StarlingX Developers +Build-Depends: debhelper-compat (= 13), + helm, + build-info, +Standards-Version: 4.5.1 +Homepage: https://www.starlingx.io + +Package: openbao-manager-helm +Section: libs +Architecture: any +Depends: ${misc:Depends} +Description: StarlingX Openbao Manager Helm Charts + This package contains helm charts for the openbao manager application. diff --git a/helm-charts/custom/openbao-manager-helm/debian/deb_folder/copyright b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/copyright new file mode 100644 index 0000000..5389f36 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/copyright @@ -0,0 +1,21 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: openbao-manager-helm +Source: https://opendev.org/starlingx/app-openbao/ + +Files: * +Copyright: (c) 2024 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. diff --git a/helm-charts/custom/openbao-manager-helm/debian/deb_folder/openbao-manager-helm.install b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/openbao-manager-helm.install new file mode 100644 index 0000000..8a0c6de --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/openbao-manager-helm.install @@ -0,0 +1 @@ +usr/lib/helm/* diff --git a/helm-charts/custom/openbao-manager-helm/debian/deb_folder/rules b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/rules new file mode 100755 index 0000000..0bb2c0e --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/rules @@ -0,0 +1,26 @@ +#!/usr/bin/make -f +export DH_VERBOSE = 1 + +export DEB_VERSION = $(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') +export PATCH_VERSION = $(shell echo $(DEB_VERSION) | cut -f 4 -d '.') +export CHART_BASE_VERSION = $(shell echo $(DEB_VERSION) | sed 's/-/./' | cut -d '.' -f 1-3) +export CHART_VERSION = $(CHART_BASE_VERSION)+STX.$(PATCH_VERSION) + +export ROOT = debian/tmp +export APP_FOLDER = $(ROOT)/usr/lib/helm + +%: + dh $@ + +override_dh_auto_build: + mkdir -p openbao-manager + cp Chart.yaml values.yaml openbao-manager + cp -r templates/ openbao-manager + make CHART_VERSION=$(CHART_VERSION) openbao-manager + +override_dh_auto_install: + # Install the app tar file. + install -d -m 755 $(APP_FOLDER) + install -p -D -m 755 openbao-manager*.tgz $(APP_FOLDER) + +override_dh_auto_test: diff --git a/helm-charts/custom/openbao-manager-helm/debian/deb_folder/source/format b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/debian/deb_folder/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/helm-charts/custom/openbao-manager-helm/debian/meta_data.yaml b/helm-charts/custom/openbao-manager-helm/debian/meta_data.yaml new file mode 100644 index 0000000..93baff6 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/debian/meta_data.yaml @@ -0,0 +1,14 @@ +--- +debname: openbao-manager-helm +debver: 1.0-1 +src_path: openbao-manager-helm +src_files: + - openbao-manager-helm/Makefile + - openbao-manager-helm/openbao-manager/templates/ + - openbao-manager-helm/openbao-manager/Chart.yaml + - openbao-manager-helm/openbao-manager/values.yaml +revision: + dist: $STX_DIST + SRC_GITREVCOUNT: + SRC_DIR: ${MY_REPO}/stx/app-openbao/helm-charts/custom/openbao-manager-helm + SRC_BASE_SRCREV: 0144b018a6a592860dace539e9fb937af7b2d26f diff --git a/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/Makefile b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/Makefile new file mode 100644 index 0000000..8f13895 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/Makefile @@ -0,0 +1,41 @@ +# +# Copyright 2017 The Openstack-Helm Authors. +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# It's necessary to set this because some environments don't link sh -> bash. +SHELL := /bin/bash +TASK := build + +EXCLUDES := doc tests tools logs tmp +CHARTS := $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.))) + +.PHONY: $(EXCLUDES) $(CHARTS) + +all: $(CHARTS) + +$(CHARTS): + @if [ -d $@ ]; then \ + echo; \ + echo "===== Processing [$@] chart ====="; \ + make $(TASK)-$@; \ + fi + +init-%: + if [ -f $*/Makefile ]; then make -C $*; fi + +lint-%: init-% + if [ -d $* ]; then helm lint $*; fi + +build-%: lint-% + if [ -d $* ]; then helm package --version $(CHART_VERSION) $*; fi + +clean: + @echo "Clean all build artifacts" + rm -f */templates/_partials.tpl */templates/_globals.tpl + rm -rf */charts */tmpcharts + +%: + @: diff --git a/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/README b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/README new file mode 100644 index 0000000..8c16b9a --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/README @@ -0,0 +1,4 @@ +This directory contains the helm chart for Openbao Manager. Rather than +being installed on the Starlingx cluster, this Openbao-Manager chart is +included within the Openbao application tarball in the stx-openbao-helm +package. diff --git a/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/Chart.yaml b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/Chart.yaml new file mode 100644 index 0000000..c34b913 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/Chart.yaml @@ -0,0 +1,10 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v2 +appVersion: "1.0.1" +description: Openbao manager helm chart +name: openbao-manager +version: 1.0.1 diff --git a/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/templates/vault-init.yaml b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/templates/vault-init.yaml new file mode 100644 index 0000000..36e0419 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/templates/vault-init.yaml @@ -0,0 +1,4008 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +apiVersion: v1 +data: + init.sh: | + #!/bin/bash + + # Get the CA path from environment vars + CERT=$CA_CERT + # Store cert as a oneliner for curl purposes + CA_ONELINE=$(awk '{printf "%s\\n", $0}' $CERT) + + # Template vaules from helm + OPENBAO_NS={{ .Release.Namespace }} + OPENBAO_NAME={{ .Values.openbao.name }} + OPENBAO_FN={{ .Values.openbao.fullname }} + HA_REPLICAS={{ .Values.server.ha.replicas }} + OPENBAO_VERSION={{ .Values.server.version }} + + # Set the domain for resolving pod names + DOMAIN="${OPENBAO_NS}.pod.cluster.local" + SVCDOMAIN="${OPENBAO_NS}.svc.cluster.local" + + # define host targets and port + POD_TARGET_BASE="$DOMAIN" # requires 'DNS NAME' of pod + ACTIVE_TARGET="${OPENBAO_FN}-active.${SVCDOMAIN}" # only the active + TARGET_PORT=8200 + + # impermanent location to store files while running + WORKDIR=/workdir + + # Health subdirectory. All openbao manager health related files + # Will be placed here. + HEALTH_SUBDIR=$WORKDIR/health + mkdir -p $HEALTH_SUBDIR + + # Selection of kubectl version from helm override + KUBECTL=kubectl + KUBECTL_HELM_OVERRIDE={{ .Values.manager.k8s.client_version }} + + # Trap and trap notification file. When SIGTERM is sent to this pod + # we want to exit promptly and gracefully. + TRAPFILE=$WORKDIR/exit_on_trap + trap "touch $TRAPFILE" SIGTERM + + # when specifying a trap for debug, remember it with this variable + # reserve trap '0' for disabling a debugging trap request + DEBUGGING_TRAP=0 + + # Pause notification file. An option to permit openbao-manager to be + # paused at any of the exit_on_trap code points. The use cases may + # include: + # - running an external procedure that should not be permitted to + # conflict with openbao-manager's operation + # - permitting time for a developer to setup conditions for debug + # - and test + PAUSEFILE=$WORKDIR/pause_on_trap + PAUSE_RATE=1 # rate at which to test for unpause + EARLY_PAUSE={{ .Values.manager.pause }} + + # Healthcheck Fail file. If this file exists then we have decided to + # force openbao manager to fail the health check + HEALTH_CHECK_FAIL=$HEALTH_SUBDIR/health_check_fail + + # Healthcheck excuses. + HEALTH_CHECK_DISABLED=$HEALTH_SUBDIR/health_check_disabled + HEALTH_EXCUSE_NETWORK=$HEALTH_SUBDIR/health_excuse_network + HEALTH_EXCUSE_INIT=$HEALTH_SUBDIR/health_excuse_init + HEALTH_EXCUSE_PAUSE=$HEALTH_SUBDIR/health_excuse_pause + + # Healthcheck excuse messages. + HC_MSG_DISABLED="Healthcheck is disabled." + HC_MSG_NETWORK="Openbao manager has initiated a network operation." + HC_MSG_INIT="Openbao manager is currently initializing." + HC_MSG_PAUSE="Openbao manager is paused for external operation." + + # Enable healthcheck excuses. + HC_DISABLE={{ .Values.manager.healthcheck.disableHC }} + HC_ENABLE_NETWORK={{ .Values.manager.healthcheck.enableNetwork }} + HC_ENABLE_INIT={{ .Values.manager.healthcheck.enableInit }} + HC_ENABLE_PAUSE={{ .Values.manager.healthcheck.enablePause }} + + # set the default manager mode; modes include + # OPENBAO_MANAGER (default) + # MOUNT_HELPER + # INTERACTIVE (i.e., when this script is sourced by an author) + if [ -z "$MANAGER_MODE" ]; then + MANAGER_MODE="OPENBAO_MANAGER" + fi + if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + MANAGER_MODE="INTERACTIVE" + fi + + # Maximum sleep seconds for mount-helper before exiting + MOUNT_HELPER_MAX_TIME=60 + + # Maximum seconds to wait for mount-helper pod to start + MAX_POD_RUN_TRIES=10 + + # Maximum seconds to wait for openbao-manager pod to exit + # Openbao-manager is not responding to SIGTERM, so will take 30 + # seconds + TERMINATE_TRIES_MAX={{ .Values.manager.waitTermination.maxTries }} + TERMINATE_TRIES_SLEEP={{ .Values.manager.waitTermination.sleepTime }} + + # Openbao key share configuration + KEY_SECRET_SHARES=5 + KEY_REQUIRED_THRESHOLD=3 + + # Enable openbao rekey upon conversion of storage from PVC to k8s + # secrets + AUTO_REKEY_CONVERT={{ .Values.manager.rekey.enableOnPVCConversion }} + + # Keep track of openbao-manager restarting the rekey procedure; if + # this variable is not true (0) and a rekey procedure is in + # progress, then openbao-manager was restarted + REKEY_STARTED=1 + + # Openbao manager will rekey the openbao at a time when the openbao + # servers are stable for a period of time specified by + # REKEY_STABLE_TIME seconds + REKEY_STABLE_TIME=300 + + # Global variable to share rekey status + REKEY_STATUS_JSON='' + + # Keep track of shards that were last successful + SHARDS_LAST_SUCCESSFUL="cluster-key" + + # Records for seal status state machine: + PODREC_F="$WORKDIR/previous_pods_status.txt" + PODREC_TMP_F="$WORKDIR/new_pods_status.txt" + + # Openbao server health query timeout during HA recovery scenario + QUERY_TMOUT={{ .Values.manager.api.healthQueryTimeout }} + + # Default curl timout for REST API commands to openbao server. + # This value is what testing shows is the default timeout. + # Specifying it explicitly for clarity. + API_TMOUT=120 + + # API timeout for unseal operations + API_UNSEAL_OP_TMOUT={{ .Values.manager.api.unsealOpTimeout }} + + # API timeout values for rekey operations + API_REKEY_QUERY_TMOUT={{ .Values.manager.api.rekeyStatusTimeout }} + API_REKEY_OP_TMOUT={{ .Values.manager.api.rekeyOpTimeout }} + + STATEFULSET_RATE=5 + INIT_CONVERGE_TIME=10 + JOIN_RATE=5 + JOIN_CONVERGE_TIME=1 + UNSEAL_RATE=10 + UNSEAL_CONVERGE_TIME=3 + STATUS_RATE={{ .Values.manager.statusCheckRate }} + if [ -z "$STATUS_RATE" ] || [ -n "${STATUS_RATE//[0-9]}" ] || \ + [ $STATUS_RATE -lt 1 ]; then + STATUS_RATE=5 + fi + + # with STATUS_RATE, the period to delay unseal + # STATUS_RATE * STATEMACH_START seconds + STATEMACH_START={{ .Values.manager.unsealWaitIntervals }} + if [ -z "$STATEMACH_START" ]; then + STATEMACH_START=3 + fi + + # Heartbeat file location + HB_FILE=$HEALTH_SUBDIR/heartbeat + + # Maximum threshold time in seconds that is allowed between + # a heartbeat call and health_check call. + HB_THRESHOLD={{ .Values.manager.healthcheck.heartbeatThreshold }} + + # Log levels + DEBUG=1 + INFO=2 + WARNING=3 + ERROR=4 + FATAL=5 + + # Default log level and the set log level (Initially set as default). + # If the log function detects an override file, then it will switch + # the set log level and then delete it. + DEFAULT_LOG_LEVEL=$INFO + LOG_LEVEL={{ .Values.manager.log.defaultLogLevel }} + LOG_OVERRIDE_FILE="$WORKDIR/log_level" + + # FUNCTIONS + + # takes major/minor version of k8s and compares + # for example: v1.28 > v1.27 > v1.26 + # + # Returns: + # 0 left is larger + # 1 equal + # 2 right is larger + function compareK8sVersion { + local left="$1" + local right="$2" + + # strip leading 'v' + left="${left#v}" + right="${right#v}" + + # compare the strings + if [ "$left" == "$right" ]; then + return 1 + fi + # compare major + if [ "${left%.*}" -gt "${right%.*}" ]; then + return 0 + elif [ "${left%.*}" -lt "${right%.*}" ]; then + return 2 + fi + + # compare the minor + if [ "${left#*.}" -gt "${right#*.}" ]; then + return 0 + fi + return 2 + } + + # Give kubectl an opportunity to express complaints in the log + function k8sComplain { + local result + + result="$( $KUBECTL version -o json 2>&1 >/dev/null )" + if [ -n "$result" ]; then + log $WARNING "kubectl: $result" + fi + } + + # Double-check that the binary exists before setting the specified + # value of KUBECTL + function switchK8sVersion { + local select="$1" + local fname="kubectl.$select" + local newbin="${KUBECTL_INSTALL_PATH}/$fname" + + which "$fname" >/dev/null + if [ $? -ne 0 -o ! -f "$newbin" ]; then + log $ERROR "Missing kubectl version: $select" + k8sComplain + return 1 + fi + + if [ "$KUBECTL" != "$fname" ]; then + KUBECTL="$fname" + log $INFO "Switching to use kubectl version $select" + fi + + k8sComplain + return 0 + } + + # Select the version of kubectl matching the running server + function pickK8sVersion { + local result + local serverver + local majorver + local minorver + local select="" + local majmin="" + local maxver + local minver + + # omit this code if the image does not support kubectl versions + if [ -z "$KUBE_VERSIONS" ]; then + k8sComplain + return + fi + + if [ -n "$KUBECTL_HELM_OVERRIDE" ]; then + # pick the binary requested, if it exists + switchK8sVersion "$KUBECTL_HELM_OVERRIDE" + if [ $? -eq 0 ]; then + return + fi + log $ERROR "kubectl version from helm-override not" \ + "available: $KUBECTL_HELM_OVERRIDE" + fi + + # use -o json for consistent usage, as oppose to --short + result="$( $KUBECTL version -o json 2>/dev/null )" + if [ $? -ne 0 ]; then + log $ERROR "Unable to get k8s server version" + # no change in value of KUBECTL + k8sComplain + return + fi + + serverver="$( jq -r '.serverVersion.gitVersion' <<<"$result" \ + | grep "[0-9]" )" + majorver="$( jq -r '.serverVersion.major' <<<"$result" \ + | grep "[0-9]" )" + minorver="$( jq -r '.serverVersion.minor' <<<"$result" \ + | grep "[0-9]" )" + if [ -z "$serverver" -o -z "$majorver" -o -z "$minorver" ]; then + log $ERROR "Unable to detect K8s server version:" \ + "["$result"]" + # no change in value of KUBECTL + k8sComplain + return + fi + + # pick matching client major/minor version + for select in $KUBE_VERSIONS noverhere; do + majmin="v${majorver}.${minorver}" + if [[ "$select" =~ ^$majmin ]]; then + break + fi + done + + if [ "$select" == noverhere ]; then + # Try to pick a near version. We really shouldn't be in + # this situation, but here is a compromise. This algorithm + # assumes that there are no omitted versions in the series + # of KUBE_VERSIONS, and that they are sorted largest to + # smallest in that list + maxver="$( awk '{print $1}' <<<"$KUBE_VERSIONS" )" + minver="$( awk '{print $NF}' <<<"$KUBE_VERSIONS" )" + + compareK8sVersion ${serverver%.*} ${maxver%.*} + if [ "$?" -le 1 ]; then + select="$maxver" + else + compareK8sVersion ${minver%.*} ${serverver%.*} + if [ "$?" -le 1 ]; then + select="$minver" + else + log $ERROR "Could not pick nearest version for kubectl" + k8sComplain + return + fi + fi + fi + + switchK8sVersion "${select%.*}" + } + + # Convert log level to text for log message + function log_to_str { + local level="$1" + local logStr + + case "$level" in + $INFO) + logStr="INFO" + ;; + $DEBUG) + logStr="DEBUG" + ;; + $WARNING) + logStr="WARNING" + ;; + $ERROR) + logStr="ERROR" + ;; + $FATAL) + logStr="FATAL" + ;; + esac + echo "$logStr" + } + + # Print the specified message to stdout if the call's specified + # level is at least the configured log level + function log { + local lvl="$1" + local logStr + local newLogLevel + + # check if log override file "Exists" + if [ -f $LOG_OVERRIDE_FILE ] \ + && [ "$MANAGER_MODE" != "INTERACTIVE" ]; then + newLogLevel=$(cat $LOG_OVERRIDE_FILE) + # validation for newLogLevel + if [[ "$newLogLevel" =~ ^[1-5]$ ]]; then + LOG_LEVEL=$newLogLevel + logStr="$( log_to_str "$LOG_LEVEL" )" + echo "$(date +%Y-%m-%dT%H-%M-%S) DEBUG" \ + "Log level set to $logStr" + else + echo "$(date +%Y-%m-%dT%H-%M-%S) DEBUG" \ + "Invalid log level read from $LOG_OVERRIDE_FILE." + fi + rm $LOG_OVERRIDE_FILE + fi + + # validate LOG_LEVEL. If it is not valid, then use + # DEFAULT_LOG_LEVEL instead. + if [[ ! "$LOG_LEVEL" =~ ^[1-5]$ ]]; then + echo "$(date +%Y-%m-%dT%H-%M-%S) DEBUG" \ + "Invalid log level detected, will be set to" \ + "$( log_to_str "$DEFAULT_LOG_LEVEL" )" + LOG_LEVEL=$DEFAULT_LOG_LEVEL + fi + + # check if the log level for this call is equal to or higher + # than the set log level + if [ "$lvl" -ge "$LOG_LEVEL" ]; then + # print log + logStr="$( log_to_str "$lvl" )" + echo "$(date +%Y-%m-%dT%H-%M-%S) $logStr ${@:2}" + fi + } + + if ! [[ "$QUERY_TMOUT" =~ ^[0-9]+$ ]]; then + log $WARNING ".Values.manager.healthQueryTimeout not an integer" + QUERY_TMOUT="" + fi + + # Check the current health status for the openbao manager. + # Return 0 if openbao manager is healthy + # Return 1 if openbao manager is unhealthy + function health_check { + local excuse_reason=() + local current_timestamp=0 + local heartbeat_timestamp=0 + local heartbeat_passed=false + local heartbeat_age=0 + + current_timestamp="$( date +%s )" + heartbeat_timestamp="$( stat -c %X $HB_FILE )" + heartbeat_age=$(( current_timestamp - heartbeat_timestamp )) + if [ $heartbeat_age -gt $HB_THRESHOLD ]; then + log $DEBUG "Heartbeat check failed" + heartbeat_passed=false + else + heartbeat_passed=true + fi + + log $DEBUG "heartbeat time: $heartbeat_age" + + if $heartbeat_passed && [ ! -f $HEALTH_CHECK_FAIL ]; then + return 0 + else + if [ "$HC_DISABLE" = "true" ] || [ -f $HEALTH_CHECK_DISABLED ]; then + excuse_reason+=("$HC_MSG_DISABLED") + elif [ "$HC_ENABLE_PAUSE" = "true" ] && [ -f $HEALTH_EXCUSE_PAUSE ]; then + excuse_reason+=("$( cat $HEALTH_EXCUSE_PAUSE )") + elif [ "$HC_ENABLE_NETWORK" = "true" ] && [ -f $HEALTH_EXCUSE_NETWORK ]; then + excuse_reason+=("$( cat $HEALTH_EXCUSE_NETWORK )") + elif [ "$HC_ENABLE_INIT" = "true" ] && [ -f $HEALTH_EXCUSE_INIT ]; then + excuse_reason+=("$( cat $HEALTH_EXCUSE_INIT )") + fi + + if [ ${#excuse_reason[@]} -gt 0 ]; then + log $INFO "Health_check fail has been excused. Reasons:" + for reason in "${excuse_reason[@]}"; do + log $INFO "$reason" + done + return 0 + else + log $INFO "Health_check has failed." + return 1 + fi + fi + } + + # Heartbeat function touches the heartbeat file to update the timestamp, + # and updates the current heartbeat timestamp + function heartbeat { + + # Do nothing if mode is not OPENBAO_MANAGER + if [ "$MANAGER_MODE" != "OPENBAO_MANAGER" ]; then + return + fi + + touch $HB_FILE + } + + # Create a health excuse file and log the reason. + function health_excuse_create { + local excuse_file_name="$1" + local excuse_reason="$2" + + # Do nothing if mode is not OPENBAO_MANAGER + if [ "$MANAGER_MODE" != "OPENBAO_MANAGER" ]; then + return + fi + + heartbeat + + # check if the requested excuse file already exists. + # If not, create the excuse file and log the message + if [ -f $excuse_file_name ]; then + log $DEBUG "The excuse file $excuse_file_name already exists." + else + echo $excuse_reason > $excuse_file_name + log $DEBUG "The excuse file $excuse_file_name created." \ + "Excuse reason: $excuse_reason" + fi + } + + # Remove the named health excuse files. + function health_excuse_remove { + local excuse_file_name="$1" + local excuse_reason + + # Do nothing if mode is not OPENBAO_MANAGER + if [ "$MANAGER_MODE" != "OPENBAO_MANAGER" ]; then + return + fi + + heartbeat + + # Check if the named excuse exists, if it is delete the file + if [ -f $excuse_file_name ]; then + excuse_reason="$( cat $excuse_file_name )" + rm $excuse_file_name + log $DEBUG "The excuse file $excuse_file_name is deleted. " \ + "The excuse reason was: $excuse_reason" + else + log $DEBUG "The excuse file $excuse_file_name is already deleted." + fi + } + + function pause_on_trap { + local thistrap="$1" + local pausenum + + if [ ! -e "$PAUSEFILE" ]; then + # no pause request + return + fi + + pausenum="$( cat "$PAUSEFILE" )" + if [ -n "$pausenum" ] \ + && [ "$pausenum" != "$thistrap" ]; then + # not on this trap + return + fi + + log $INFO "Openbao manager is paused ($thistrap)" + health_excuse_create "$HEALTH_EXCUSE_PAUSE" "$HC_MSG_PAUSE" + # Until pause file is removed by the author, + # or until the content of pause_on_trap file is + # not-empty and not matching the current trap. + # + # If the pause_on_trap file containing specific trap number is + # replaced with empty file: the pause state is maintained. + while [ -e "$PAUSEFILE" ]; do + pausenum="$( cat "$PAUSEFILE" )" + if [ -n "$pausenum" ] \ + && [ "$thistrap" != "$pausenum" ]; then + break; + fi + sleep "$PAUSE_RATE" + done + health_excuse_remove "$HEALTH_EXCUSE_PAUSE" + log $INFO "Openbao manager is unpaused" + } + + function exit_on_trap { + local trap="$1" + local tfnum="" + + if [ "$MANAGER_MODE" == "INTERACTIVE" ]; then + # do not interfere with exit_on_trap intended for + # openbao-manager pod + return + fi + + heartbeat + + # Debug option pause_on_trap + pause_on_trap "$trap" + + if [ -e "$TRAPFILE" ]; then + tfnum=$(cat $TRAPFILE) + log $DEBUG "exit_on_trap: removing $TRAPFILE" + rm "$TRAPFILE" # for workdir on PVC + if [ -z "$tfnum" ]; then + # an empty trap file is the default expected behaviour + log $INFO "exit_on_trap: ($trap)" + exit + # handle trap debugging feature - a developer specifies the + # trap number to target a specific exit_on_trap call. + # Setting a value of 0 (zero) disables the debugging trap + elif [ "$tfnum" -eq 0 ]; then + log $DEBUG "exit_on_trap: ($trap):" \ + "disable debug trap ($DEBUGGING_TRAP)" + DEBUGGING_TRAP=0 + # there is no trap with value zero + return + else + DEBUGGING_TRAP="$tfnum" + log $DEBUG "exit_on_trap: ($trap): " \ + "enable debug trap ($DEBUGGING_TRAP)" + # check now just in case it matches + if [ "$DEBUGGING_TRAP" -eq "$trap" ]; then + log $INFO "exit_on_trap: ($trap): matching" + exit + fi + fi + # check if there is a matching debug trap set + elif [ "$DEBUGGING_TRAP" -eq "$trap" ]; then + log $INFO "exit_on_trap: ($trap): matching" + exit + else + log $DEBUG "exit_on_trap: ($trap): no trap file, no exit" + fi + } + + # splits keys into separate files. Each file contains the key and the base64 encoded version. + # root token will be stored separately + + function splitShard { + local index="$1" + jq '{"keys": [.keys['$index']], "keys_base64": [.keys_base64['$index']]}' + } + + # merges two split keys + function mergeKeyJson { + # the two parameters are names for variables + local jstr1="$1" + local jstr2="$2" + + mkfifo "$WORKDIR"/s1 + mkfifo "$WORKDIR"/s2 + + ( + jq -Mn --argfile file1 $WORKDIR/s1 --argfile file2 $WORKDIR/s2 ' + def mergek: ($file1, $file2) | .keys as $k | $k; + def mergeb: ($file1, $file2) | .keys_base64 as $b | $b; + {keys: (reduce mergek as $x ([]; . + $x)), + keys_base64: (reduce mergeb as $x ([]; . + $x))} + ' & ) 2>/dev/null + + echo -n "${!jstr1}" > "$WORKDIR"/s1 + echo -n "${!jstr2}" > "$WORKDIR"/s2 + + rm -f "$WORKDIR"/s1 "$WORKDIR"/s2 + } + + # Prepare a json document from the k8s secrets prefixed with + # prefix, and the root token + # + # Required parameter: The prefix of the k8s secrets containing + # the shards + # + # Outputs the json document which is comparable to the original + # response for openbao initialization. The calling function is + # responsible for validating the document content. + # + function reconstructInitResponse { + local prefix="$1" + local index + local keys + local mkeys + + # pull secrets from k8s and merge into one json file. + for index in $( seq 0 $(( KEY_SECRET_SHARES - 1 )) ); do + keys="$( get_secret "${prefix}-$index" )" + if [ "$index" -eq 0 ]; then + mkeys="$keys" + continue + fi + mkeys=$( mergeKeyJson mkeys keys ) + done + + # append the root secret and echo the document + echo "$mkeys" | jq -c '{keys: .keys, + keys_base64: .keys_base64, + root_token: "'$( get_secret "cluster-key-root" )'"}' + } + + # Check the structure of json data and confirm equivalence of + # the stdin with stored secrets + # + # Required parameter: The prefix of the k8s secrets containing + # the shards in stored secrets + # + # Returns the normal linux success=0, failure!=0 + function validateSecrets { + local keyprefix="$1" + local text + local keys + local keys_base64 + local root_token + local count + local saved + local shaA + local shaB + + text=$( cat ) + keys=$( echo "$text" | jq '.keys' ) + keys_base64=$( echo "$text" | jq '.keys_base64' ) + root_token=$( echo "$text" | jq -r '.root_token' ) + # response is 'null' if the dict key is missing + # response is empty (-z) is the source document is empty + if [ -z "$keys" -o "$keys" == "null" \ + -o -z "$keys_base64" -o "$keys_base64" == "null" \ + -o -z "$root_token" -o "$root_token" == "null" ]; then + log $ERROR "one or more missing keys" + return 1 + fi + + count=$( echo "$keys" | jq '. | length' ) + if [ $? -ne 0 ]; then + log $ERROR "jq did not parse keys length" + return 1 + fi + if [ -z "$count" ] || [ "$count" -ne "$KEY_SECRET_SHARES" ]; then + log $ERROR "Incorrect array length for keys:" \ + "$count instead of $KEY_SECRET_SHARES" + return 1 + fi + count=$( echo "$keys_base64" | jq '. | length' ) + if [ $? -ne 0 ]; then + log $ERROR "jq did not parse keys_base64 length" + return 1 + fi + if [ -z "$count" ] || [ "$count" -ne "$KEY_SECRET_SHARES" ]; then + log $ERROR "Incorrect array length for keys_base64:" \ + "$count instead of $KEY_SECRET_SHARES" + return 1 + fi + + saved="$( reconstructInitResponse "${keyprefix}" )" + + # finally ensure that the saved secrets are the same as the + # supplied text + shaA=$( echo "$text" | sha256sum ) + shaB=$( echo "$saved" | sha256sum ) + if [ "$shaA" != "$shaB" ]; then + log $ERROR "saved data differs from source data" + return 1 + fi + + log $INFO "Verified stored secrets are the same as supplied data" + return 0 + } + + # Creates a list of all k8s openbao pods and stores in text file. + # Converts ips from X.X.X.X or a:b:c::d to X-X-X-X for use as pod + # dns names + # + # Optional parameter: + # --ha : append openbao server active/standby status (boolean) + # + # Example output with --ha + # stx-openbao-0 172-16-226-97 true + function getOpenbaoPods { + local ha="$1" + local jpath + local meta='{.metadata.name}' + local ip='{.status.podIPs[].ip}' + local active='{.metadata.labels.openbao-active}' + local jfields=${meta}'{"\t"}'${ip} + + if [ "$ha" == "--ha" ]; then + jfields=${jfields}'{"\t"}'${active} + fi + jpath='{range .items[*]}'"$jfields"'{"\n"}{end}' + + $KUBECTL get pods \ + -n "$OPENBAO_NS" \ + -l component=server,app.kubernetes.io/name=openbao \ + -o=jsonpath="$jpath" \ + | sed 's/\.\|:/-/g' + } + + # Wait for the openbao servers in the stateful set to be + # created before initializing + function waitForPods { + local jsonPath='{range .items[*]}{.metadata.name}{"\t"} \ + {.status.podIPs[].ip}{"\t"}{.status.phase}{"\n"} \ + {end}' + + CURRENT_PODS=$($KUBECTL get pods \ + -l component=server,app.kubernetes.io/name=openbao \ + -o=jsonpath="$jsonPath" \ + | grep Running \ + | wc -l) + DESIRED_PODS=$1 + + if ! [[ "$CURRENT_PODS" =~ ^[0-9]+$ ]]; then + log $ERROR "Invalid Running pod number ($CURRENT_PODS) from kubectl get pods" + CURRENT_PODS=0 + fi + + while [ $CURRENT_PODS -lt $DESIRED_PODS ]; do + sleep "$STATEFULSET_RATE" + log $INFO "Waiting for ${OPENBAO_FN}" \ + "statefulset running pods ($CURRENT_PODS) to equal" \ + "desired pods ($DESIRED_PODS)" + CURRENT_PODS=$($KUBECTL get pods \ + -l component=server,app.kubernetes.io/name=openbao \ + -o=jsonpath="$jsonPath" \ + | grep Running \ + | wc -l) + done + } + + # Takes the json document output from openbao initialization + # and stores it into secrets for key shards and the root token + # + # Required parameter: The prefix of the k8s secrets into which to + # store the shards + # + # This only works if the secrets are not pre-existing. An error + # is printed by set_secrets. + function storeOpenbaoInitSecrets { + local keyprefix="$1" + local secrets + local index + local split_json + + secrets=$( cat ) + + for index in $(seq 0 $((KEY_SECRET_SHARES - 1 ))); do + split_json=$( echo -n "$secrets" | splitShard "$index" ) + set_secret "${keyprefix}-$index" /dev/stdin <<< "$split_json" + done + + # if the data contains root_token, save it as well + split_json=$( echo "$secrets" | jq -r '.root_token' ) + if [ -n "$split_json" -a "$split_json" != 'null' ]; then + set_secret "${keyprefix}-root" /dev/stdin <<< "$split_json" + fi + } + + # Address a openbao server with REST API request. Capture stderr, + # stdout and result of curl commands. Print error and debug logs + # + # Required positional parameters, in order: + # Response variable : variable in which to store the response + # from openbao + # http request type : GET, POST, DELETE + # openbao server : FQDN + # openbao REST API path : e.g., /sys/health + # + # Optional final parameter : a quoted string of data + # + # Examples: + # # get health status query for the active openbao status + # openbaoAPI myvar GET $ACTIVE_TARGET /sys/health + # + # # post rekey initialization with shares 5 and threshold 3 + # data='{"secret_shares": 5,"secret_threshold": 3}' + # openbaoAPI myvar POST $ACTIVE_TARGET /sys/rekey/init "$data" + # + # Overridable ENV variables: + # API_TMOUT: the curl timeout + # NO_HEADER: omit header (the root token) if not empty + # + # Output: + # Return the stdout and command result code + # + # Print log messages for errors. The responses from openbao are + # restricted to DEBUG lovel log in case there's secret information + # in them. But a non-specific ERROR message is printed in all + # cases of errors. + function openbaoAPI { + local answer="$1" + local reqarg="$2" + local server="$3" + local apipath="$4" + local data="$5" + local cmderr="" + local cmdout="" + local cmdres=1 + local header="" + local errors="" + + if [ -z "$NO_HEADER" ]; then + header="X-Vault-Token:$( get_secret cluster-key-root )" + fi + + log $DEBUG "Executing: [curl -s -S --cacert \""$CERT"\"" \ + ${API_TMOUT:+"--connect-timeout" "$API_TMOUT"} \ + ${header:+"--header" "xxxx"} \ + "--request \"$reqarg\"" \ + ${data:+"--data" "xxxx"} \ + "\"https://${server}:${TARGET_PORT}/v1${apipath}\"]" + + health_excuse_create "$HEALTH_EXCUSE_NETWORK" "$HC_MSG_NETWORK" + # Capture stderr and stdout copied from google search example + # on stack overflow. Add capture of the command result code + { + IFS=$'\n' read -r -d '' cmderr; + IFS=$'\n' read -r -d '' cmdout; + cmdres="$( echo "$cmdout" | tail -n1 )" + cmdout="$( echo "$cmdout" | head -n-1 )" + } < <((printf '\0%s\0' "$( + curl -s -S --cacert "$CERT" \ + ${API_TMOUT:+"--connect-timeout" "$API_TMOUT"} \ + ${header:+"--header" "$header"} \ + --request "$reqarg" \ + ${data:+"--data" "$data"} \ + "https://${server}:${TARGET_PORT}/v1${apipath}" + echo "$?" + )" 1>&2) 2>&1) + + health_excuse_remove "$HEALTH_EXCUSE_NETWORK" + + if [ "$cmdres" -ne 0 ]; then + log $ERROR "curl returns non-zero result: $cmdres" + fi + if [ -n "$cmderr" ]; then + log $ERROR "curl returns stderr" + log $DEBUG "curl returns stderr: [$cmderr]" + fi + + if [ -n "$cmdout" ]; then + # errors from the REST API + errors=$( echo "$cmdout" | jq -cr '.errors' ) + if [[ "$errors" != 'null' ]] && [ -n "$errors" ]; then + log $ERROR "openbao REST API error" + log $DEBUG "openbao REST API error: $errors" + if [ "$cmdres" -eq 0 ]; then + # this code wants to know if there was an error + cmdres=1 + fi + fi + fi + eval "$answer"='$cmdout' + return $cmdres + } + + # Initializes the first openbao pod, only needs to be performed once + # after deploying the helm chart + # Stores the root token and master key shards in k8s secrets + function initOpenbao { + local V0 # the zeroeth openbao pod + local keys + local key_error + local shares + local threshold + + V0=$(awk 'NR==1{print $2}' $WORKDIR/pods.txt) + log $INFO "Initializing $V0" + shares='"secret_shares": '$KEY_SECRET_SHARES + threshold='"secret_threshold": '$KEY_REQUIRED_THRESHOLD + + NO_HEADER=true \ + openbaoAPI keys POST $V0.$POD_TARGET_BASE \ + /sys/init "{$shares, $threshold}" + + key_error=$(echo -n "$keys"| jq -r '.errors[]?') + if [ -n "$key_error" ]; then + log $ERROR "openbao init request failed: $key_error" + fi + + echo "$keys" | storeOpenbaoInitSecrets cluster-key + + # check if the secrets match openbao's REST API response + echo "$keys" | validateSecrets cluster-key + } + + # Uses the master key shards to unseal openbao + function unsealOpenbao { + local server="$1" + local prefix="$2" + local index + local b64key + local data + local response + local value + local autherror + + if [ -z "$prefix" ]; then + prefix='cluster-key' + fi + + # always abort an unseal in progress + data='{"reset": true}' + NO_HEADER=true \ + API_TMOUT=$API_UNSEAL_OP_TMOUT \ + openbaoAPI response POST $server.$POD_TARGET_BASE \ + /sys/unseal "$data" + if [ $? -ne 0 ]; then + # error is already printed + # Including if openbao is already unsealed. + if [[ "$response" == *"openbao is unsealed"* ]]; then + log $WARNING "unsealOpenbao: server $server is" \ + "already unsealed" + fi + return 1 + fi + + for index in $(seq 0 $((KEY_SECRET_SHARES - 1 ))); do + b64key=$( get_secret "${prefix}-$index" \ + | jq -r '.keys_base64[]' ) + data="{\"key\": \"$b64key\"}" + + NO_HEADER=true \ + API_TMOUT=$API_UNSEAL_OP_TMOUT \ + openbaoAPI response POST $server.$POD_TARGET_BASE \ + /sys/unseal "$data" + if [ $? -ne 0 ]; then + # error is already printed, including errors from the + # openbao REST API; but for debugging purposes, highlight + # the authentication error + autherror="cipher: message authentication failed" + if [[ "$response" == *"$autherror"* ]]; then + log $ERROR "Failed to authenticate /sys/unseal" \ + "with $prefix" + # perhaps use this info in the future + return 2 + fi + log $DEBUG "Unknown failure authenticating unseal" \ + "$response" + return 1 + fi + + # when the unseal completes with KEY_REQUIRED_THRESHOLD then + # the response will indicate sealed=false + value="$( echo "$response" | jq -r ".sealed" )" + if [ "$value" == "false" ]; then + log $DEBUG "Success authenticating unseal" + return 0 + fi + + value="$( echo "$response" | jq -r ".progress" )" + log $DEBUG "Success authenticating unseal" \ + "(${value}/${KEY_REQUIRED_THRESHOLD})" + # Some sleep is required to allow Raft convergence + sleep "$UNSEAL_CONVERGE_TIME" + done + + log $ERROR "unsealOpenbao completes without unseal or error" + return 1 + } + + # Unseal a openbao server under conditions of recovery, + # including selecting and remembering alternate shard + # secrets. + # + # This algorithm remembers the last shards used to unseal the openbao, + # to prioritize using those again the next time. + function unsealOpenbaoRecover { + local server="$1" + local attempted + local use_secrets="" + + if [ -n "$SHARDS_LAST_SUCCESSFUL" ]; then + # double check the keys we were using are not deleted + if assertShardSecrets "$SHARDS_LAST_SUCCESSFUL"; then + use_secrets="$SHARDS_LAST_SUCCESSFUL" + fi + fi + + use_secrets="$use_secrets $( \ + getOtherShardSecrets "$SHARDS_LAST_SUCCESSFUL" )" + for attempted in $use_secrets; do + log $INFO "Attempt unseal with $attempted" + unsealOpenbao "$server" "$attempted" + case $? in + 0) + SHARDS_LAST_SUCCESSFUL="$attempted" + return 0 + ;; + 2) + # an error is already printed + # try a different set of shards + continue + ;; + *) + # failure is not clear, try again later + log $ERROR "Fail to unseal $server with" \ + "$attempted; try later" + return 1 + ;; + esac + done + + log $ERROR "No set of shards unseal the server $server:" \ + "attempted: $use_secrets" + return 1 + } + + # Takes the address of openbao-0 as the cluster leader and + # joins other nodes to raft + function joinRaft { + local dnsname="$1" + local activeLink="https://${ACTIVE_TARGET}:${TARGET_PORT}" + local dataJson="{\"leader_api_addr\": \"$activeLink\", \"leader_ca_cert\": \"$CA_ONELINE\"}" + RAFT_STATUS="" + while [ "$RAFT_STATUS" != "true" ]; do + + openbaoAPI RAFT_STATUS POST $dnsname.$POD_TARGET_BASE \ + /sys/storage/raft/join "$dataJson" + + log $INFO "$dnsname $RAFT_STATUS" + RAFT_STATUS=$(echo $RAFT_STATUS | jq -r .joined) + sleep "$JOIN_CONVERGE_TIME" + done + } + + function runStateMachine { + local host="$1" + local dns_name="$2" + local sealed="$3" + local status_rec + local old_rec + local counter + + status_rec="/$host/$dns_name/$sealed/" + + # log compression: do not print logs when status is unchanged + # omit counter when checking openbao server state change + old_rec="$( grep "$status_rec" "$PODREC_F" )" + if [ $? -ne 0 ]; then + log $DEBUG "$( grep "$dns_name" $WORKDIR/pods.txt )" + log $INFO "Sealed status of $dns_name is now: $sealed" + + # reread the record by hostname only + old_rec="$( grep "^/$host/" "$PODREC_F" )" + else + log $DEBUG "There is no change in pod seal status" + fi + + if [ "$sealed" != "true" ]; then + # There is nothing more to do: the openbao is unsealed + # or the sealed status is unclear + echo "$status_rec" >> "$PODREC_TMP_F" + return + fi + + # The openbao is sealed + # + # Check if there is a countdown in progress + # + # else -z old_rec: "the pod didn't have an IP address the last + # iteration, but now it does" - treat the same as "sealed + # without a countdown" + counter="" + if [ -n "$old_rec" ]; then + counter="$( echo "$old_rec" | awk -F/ '{print $5}' )" + fi + + if [ -z "$counter" ]; then + # sealed without a countdown: start counting + log $DEBUG "Sealed openbao $host: begin unseal delay:" \ + "$( expr "$STATUS_RATE" \* "$STATEMACH_START" )s" + echo "${status_rec}${STATEMACH_START}" >> "$PODREC_TMP_F" + return + fi + + # Check for end of period: 1 means "zero at this interval" + # "less than 1" for resilience + if [ "$counter" -le 1 -o "$STATEMACH_START" -eq 0 ]; then + # We've waited (STATUS_RATE * STATEMACH_START) seconds + # Or, STATEMACH_START == 0 means do not delay + log $INFO "Unsealing $dns_name" + unsealOpenbaoRecover "$dns_name" + echo "$status_rec" >> "$PODREC_TMP_F" + return + fi + + # finally, continue to countdown + counter="$( expr "$counter" - 1 )" + echo "${status_rec}${counter}" >> "$PODREC_TMP_F" + } + + function openbaoInitialized { + local response + local dnsname + local initialized + local text + + # Wait for the pod to respond with a positive openbao API response + # (i.e., not just a curl failure, and not a openbao API failure) + while true; do + dnsname=$(awk 'NR==1{print $2}' $WORKDIR/pods.txt) + if [ -z "$dnsname" ]; then + log $INFO "waiting..." + sleep $STATUS_RATE + getOpenbaoPods > $WORKDIR/pods.txt + continue + fi + + log $INFO "Query server $dnsname for initialization status" + NO_HEADER=true \ + API_TMOUT=$QUERY_TMOUT \ + openbaoAPI response GET $dnsname.$POD_TARGET_BASE /sys/health + if [ $? -ne 0 ]; then + log $INFO "waiting..." + sleep $STATUS_RATE + getOpenbaoPods > $WORKDIR/pods.txt + continue + fi + break + done + + echo -n "$response" > $WORKDIR/healthcheck.txt + initialized=$( echo "$response" | jq -r .initialized ) + + text="$( grep $dnsname $WORKDIR/pods.txt )" + if [ $? -eq 0 ]; then + log $DEBUG "$text" + log $DEBUG "Initialized status is $initialized" + fi + + # The empty check is here as a extra safety net, but an + # investigation into in which exact conditions the result would + # be empty would be helpful. + if [ ! -z $initialized ] && [ $initialized = false ]; then + return 1 + else + return 0 + fi + } + + function set_secret { + local secret="$1" + local contentf="$2" + local output + local result + + output="$( $KUBECTL create secret generic -n "$OPENBAO_NS" \ + "$secret" "--from-file=strdata=$contentf" 2>&1 )" + result=$? + if [ "$result" -ne 0 ]; then + log $ERROR "Failed to create secret $secret" + log $DEBUG "Output: [$output]" + fi + return $result + } + + function get_secret { + local secret="$1" + + $KUBECTL get secrets -n "$OPENBAO_NS" "$secret" \ + -o jsonpath='{.data.strdata}' \ + | base64 -d + } + + # When openbao-manager is run in "MOUNT_HELPER" mode, this function + # will not return. Instead the function will exit_on_trap or exit + # when it times-out. + # + # Basically: this function doesn't do anything except wait to be + # terminated. + # + # Openbao-manager in MOUNT_HELPER has PVC mounted, allowing the real + # openbao-manager to read secrets from cluster_keys.json + function mountHelper { + local count + + # omit this function if this pod is not the mount helper + if [ -z "$MANAGER_MODE" -o "$MANAGER_MODE" != "MOUNT_HELPER" ]; then + log $INFO "Mode is OPENBAO_MANAGER" + return + fi + + # When openbao-manager is running in this mode, it should be + # deleted by openbao-manager running in the default mode, which + # is using this pod to read secrets from mounted PVC + log $INFO "Mode is $MANAGER_MODE" + + # start with some debug/error logs + if [ -f "$PVC_DIR/cluster_keys.json" ]; then + log $DEBUG "Successfully mounted secrets file" + else + log $WARNING "Secrets file not found" + fi + + # sleep for MOUNT_HELPER_MAX_TIME, expecting SIGTERM signal + log $INFO "Waiting for termination request via SIGTERM" + count=0 + while [ "$count" -lt "$MOUNT_HELPER_MAX_TIME" ]; do + exit_on_trap + count=$((count+1)) + sleep 1 + done + + # Normally should exit by exit_on_trap, but here we timeout + # waiting for the real openbao-manager to delete this job/pod. + log $INFO "Exiting without receiving SIGTERM request" + exit 0 + } + + # Check if a secret exists + # + # Returns the normal linux success=0, failure!=0 + # Prints the name of the secret + function secretExists { + local name="$1" + $KUBECTL get secrets -n "$OPENBAO_NS" "$name" \ + -o jsonpath='{.metadata.name}' 2>/dev/null \ + | grep "$name" + } + + # Return linux success=0 if any of the secrets exist + function secretsExistAny { + local list="$@" + local name + + for name in $list; do + secretExists $name >/dev/null + if [ $? -eq 0 ]; then + return 0 + fi + done + + return 1 + } + + # Assert that the shard secrets starting with prefix exist + # + # Parameter: prefix for k8s secrets, such as 'cluster-key' + # + # Optional second parameter: + # --nokeys : failed if at least one exists + # + # Returns the normal linux success=0, failure!=0 + # + # When --nokeys is selected, the failure return code is the number + # of secrets found. Zero secrets were expected. + # + # When --nokeys is omitted, the failure return code is either the + # number of secrets found or if the number of secrets found was + # zero, KEY_SECRET_SHARES is returned as error code + function assertShardSecrets { + local prefix="$1" + local nokey="$2" + local i + local count=0 + + for i in $( seq 0 $((KEY_SECRET_SHARES-1)) ); do + secretExists "${prefix}-$i" >/dev/null + if [ $? -eq 0 ]; then + count=$((count+1)) + fi + done + if [ "$nokey" == "--nokeys" ]; then + # 0 secrets == true (0) + # Else return the number of secrets + return $count + fi + if [ "$count" -eq "$KEY_SECRET_SHARES" ]; then + return 0 + elif [ "$count" -eq 0 ]; then + return "$KEY_SECRET_SHARES" # an error result + fi + return "$count" + } + + # Return a list of other existing Shard secrets other than the set + # specified + # + # Sort by priority order: + # cluster-key + # cluster-rekey + # cluster-key-bk + # + function getOtherShardSecrets { + local omit="$1" + local secrets="cluster-key cluster-rekey cluster-key-bk" + local secret + local others="" + + for secret in $secrets; do + if [ "$secret" == "$omit" ]; then + continue + fi + if assertShardSecrets $secret; then + others="$others $secret" + fi + done + echo $others + } + + # Delete the specified list of secrets + # + # Uses a single kubectl command + function deleteSecrets { + local secrets="$@" + local text + text="$( $KUBECTL delete secrets -n "$OPENBAO_NS" \ + $secrets 2>&1 )" + if [ $? -ne 0 ]; then + log $ERROR "Error deleting secrets: ["$text"]" + return 1 + fi + log $INFO $text + return 0 + } + + # Check if the PVC resource exists + # + # Returns 0 if pvc does not exist + # Returns 1 if pvc exists but is terminating + # Returns 2 if pvc exists and is not terminating + # Prints the name of the PVC resource + function pvcRemoved { + local text + local jqscript + + jqscript='.items + | map(select(.metadata.name | test("^manager-pvc"))) + | "\(.[0].metadata.name) \(.[0].status.phase)"' + + # using jq since kubernetes does not support regex + # the grep makes sure the result contains the 'manager-pvc' + # string (as opposed to 'null' for example) + text="$( + $KUBECTL get persistentvolumeclaims -n "$OPENBAO_NS" -o json \ + | jq -r "$jqscript" 2>/dev/null \ + | grep manager-pvc )" + + if [ -n "$text" ]; then + readarray -d " " -t pvcInfo <<< "$text" + pvcName="${pvcInfo[0]}" + pvcStatus="${pvcInfo[1]}" + echo "$pvcName" + if [ "$pvcStatus" = "Terminating" ]; then + return 1 + else + return 2 + fi + fi + + return 0 + } + + # Check if the PVC is mounted to any pod in openbao namespace + # + # Returns the normal linux success=0, failure!=0 + # Prints the name of the PVC resource + function testPVCMount { + local result + local cspec + local vspec + + cspec=".items[*].spec.containers[*]" + vspec="volumeMounts[?(@.name=='manager-pvc')].name" + + # this kubectl query returns zero whether manager-pvc is + # found or not + # result variable is either empty or 'manager-pvc' + result="$( $KUBECTL get pods -n "$OPENBAO_NS" \ + -o jsonpath="{${cspec}.${vspec}}" )" + + if [ -n "$result" ]; then + return 0 + fi + return 1 # assertion 'fails' + } + + # This function prints a DEBUG log of kubectl delete + function deleteMountHelper { + local text + local result + + log $DEBUG "Waiting for delete of mount-helper job" + text="$( $KUBECTL delete --ignore-not-found=true --wait=true \ + -f /opt/yaml/pvc-attach.yaml 2>&1 )" + result=$? + log $DEBUG "Output of deleting mount-helper: [$text]" + return $result + } + + # Run shred on the file content of PVC + # + # All files a shredded, and the result is an error if + # - command return code is non-zero + # - file comparison shows unchanged file(s) + # + # A warning is issued if shred/kubectl command has any stdout or + # stderr + # + # Returns the normal linux success=0, failure!=0 + function securelyWipePVC { + local helper="$1" + + if [ -z "$helper" ]; then + log $ERROR "No pod specified for shredding" + return 1 + fi + + # get profile of the files before shredding + $KUBECTL exec -n "$OPENBAO_NS" "$helper" -- \ + bash -c 'find /mnt/data -type f \ + | sort | xargs wc | head -n-1' \ + >/tmp/shred_before.txt 2>&1 + log $DEBUG "Original files: [$( cat /tmp/shred_before.txt )]" + + # run the shred command + # + # Shred all the files in mounted /mnt/data/ + # + # The shred by default has three randomized passes, and with -z + # option will finalize with zeros. -f prompts shred to work + # around any unexpected file permissions + text="$( $KUBECTL exec -n "$OPENBAO_NS" "$helper" -- \ + bash -c '\ + result=0; \ + while read fname; do \ + shred -f -z "$fname"; \ + [ $? -ne 0 ] && result=1; \ + done <<<"$(find /mnt/data -type f )"; \ + exit $result' 2>&1 )" + result=$? + + # get profile of the files after shredding + $KUBECTL exec -n "$OPENBAO_NS" "$helper" -- \ + bash -c 'find /mnt/data -type f \ + | sort | xargs wc | head -n-1' \ + >/tmp/shred_after.txt 2>&1 + log $DEBUG "Shredded files: [$( cat /tmp/shred_after.txt )]" + + # compare the profiles for error reporting + # + # If the file lists, pushed through wc, have files with the same + # character, word, and line counts then report an error: a file + # has not been shred + # + # Ignore files that were empty + difftext="$( diff -wuU100000 /tmp/shred_before.txt \ + /tmp/shred_after.txt )" + unchanged="$( echo "$difftext" | grep "^ " \ + | grep -v "^\([ ]\{1,\}0\)\{3\} /" )" + + # Report the errors/success + if [ "$result" -ne 0 ]; then + log $ERROR "Error on shred: [$text]" + if [ -n "$unchanged" ]; then + log $ERROR "Unchanged: [$unchanged]" + fi + return 1 + fi + if [ -n "$text" ]; then + log $WARNING "Output of shred is not empty: [$text]" + fi + if [ -n "$unchanged" ]; then + log $ERROR "Shred did not shred some files" + log $ERROR "Unchanged: [$unchanged]" + return 1 + fi + + log $INFO "Shredding of PVC data verified" + + return 0 + } + + # Delete the PVC resource + # + # The delete will succeed even if attached to a pod, such as a + # terminating openbao-manager or mount-helper - the PVC remains + # in terminating status until the pod is also terminated. + function deletePVC { + local text + local name + + name="$( pvcRemoved )" + if [ $? -eq 2 ] && [[ "$name" =~ ^manager-pvc ]]; then + text="$( $KUBECTL delete persistentvolumeclaims \ + -n "$OPENBAO_NS" "$name" 2>&1 )" + if [ $? -ne 0 ]; then + log $ERROR "Error deleting PVC: [$text]" + else + log $INFO "$text" + fi + else + log $WARNING "Request to delete PVC but PVC not found" + fi + } + + # Run a job/pod, to mount the PVC resource, and retrieve the secrets + # from PVC. + # + # See also the function mountHelper and the ConfigMap named: + # {{ .Values.openbao.name }}-mount-helper + # + # This function does not support overwriting an existing + # cluster-key-* secret, but it does support validating those secrets + # if they exist + function convertPVC { + local output + local pod + local count + local text + local PVCtext + local result + local waitPVCterm + + if testPVCMount; then + log $ERROR "Cannot mount PVC already mounted" + return 1 + fi + + # run the pod + output="$( $KUBECTL apply -f /opt/yaml/pvc-attach.yaml 2>&1 )" + if [ $? -ne 0 ]; then + log $ERROR "Failed to apply mount-helper" + log $DEBUG "Output: [$output]" + deleteMountHelper + return 1 + fi + + # wait for pod + pod='' + count=0 + log $INFO "Waiting for mount-helper pod to run" + while [ -z "$pod" -a "$count" -le "$MAX_POD_RUN_TRIES" ]; do + count=$((count+1)) + text="$( $KUBECTL get pods -n "$OPENBAO_NS" \ + | grep "mount-helper" )" + pod="$( echo "$text" | grep "Running" | awk '{print $1}' )" + if [ -z "$pod" ]; then + sleep 1 + fi + done + + if [ -z "$pod" ]; then + log $ERROR "Failed to run mount-helper pod" + log $DEBUG "Pod state: [$( echo $text )]" + deleteMountHelper + return 1 + fi + + # get the pvc data + PVCtext="$( $KUBECTL exec -n "$OPENBAO_NS" "$pod" \ + -- cat /mnt/data/cluster_keys.json )" + if [ $? -ne 0 -o -z "$PVCtext" ]; then + log $ERROR "Failed to read cluster_keys.json" + deleteMountHelper + return 1 + fi + log $INFO "Data retrieved from PVC" + + # if the Root secret is pre-existing, compare the existing + # shard secrets and root secret before deleting the PVC + $KUBECTL get secrets -n "$OPENBAO_NS" \ + cluster-key-root >/dev/null 2>&1 + if [ $? -eq 0 ]; then + log $INFO "Cluster secrets exist:" \ + "validating" + else + # create a secret from the data + echo "$PVCtext" | storeOpenbaoInitSecrets cluster-key + fi + + # verify the data stored versus text from PVC + echo "$PVCtext" | validateSecrets cluster-key + result=$? + if [ "$result" -eq 0 ]; then + securelyWipePVC "$pod" + # omit deleting the PVC for manual analysis and shred + # when the wipe fails + if [ $? -eq 0 ]; then + deletePVC + fi + fi + + # clean up but do not care about the result + deleteMountHelper + + # Sleep before finishing conversion, so that pvc termination process has started + waitPVCterm=5 + sleep $waitPVCterm + + return $result + } + + function convertBootstrapSecrets { + local text + local count + + text="$( get_secret cluster-key-bootstrap )" + echo "$text" | storeOpenbaoInitSecrets cluster-key + + # verify the split secrets versus the bootstrap text + echo "$text" | validateSecrets cluster-key + if [ $? -ne 0 ]; then + # an error is already printed + return 1 + fi + + deleteSecrets cluster-key-bootstrap + + # Also validate and delete the PVC resource + # This procedure depends on waiting for the old version + # of openbao-manager pod to exit + count="$TERMINATE_TRIES_MAX" + log $INFO "Waiting for openbao-manager pod to exit" + while testPVCMount && [ "$count" -gt 0 ]; do + sleep "$TERMINATE_TRIES_SLEEP" + count=$((count-1)) + done + + if [ $count -eq 0 ]; then + log $WARNING "Maximum time reached waiting" \ + "for the previous pod to be terminated." + fi + + convertPVC + } + + # When enabled, after conversion of storage from PVC to k8s secrets, + # Openbao-manager will prompt itself to rekey the openbao server + # storage. + function requestRekey { + local value + + if [ "$AUTO_REKEY_CONVERT" != "true" ]; then + return + fi + log $INFO "Auto rekey enabled: [$AUTO_REKEY_CONVERT]" + + secretExists cluster-rekey-request >/dev/null + if [ $? -eq 0 ]; then + value="$( get_secret cluster-rekey-request )" + log $WARNING "Auto rekey: rekey request exists: $value" + return + fi + + value=$( uuidgen ) + set_secret cluster-rekey-request /dev/stdin <<<"$value" + if [ $? -eq 0 ]; then + log $INFO "Rekey requested: $value" + else + log $ERROR "Failed to request rekey: $value" + fi + return + } + + function runConversion { + if [ -n "$K8S_SECRETS_PREEXIST" ]; then + log $INFO "Cluster secrets exist" + return + elif [ -n "$BOOTSTRAP_PREEXISTS" ]; then + # this is the normal application update procedure; the + # lifecycle code retrieved the secrets from previous version + # of the application. + log $INFO "Using secrets provided in $BOOTSTRAP_PREEXISTS" + convertBootstrapSecrets + requestRekey + return + elif [ -z "$PVC_PREEXISTS" ]; then + log $INFO "No pre-existing secrets exist" + return + fi + + # Finally, read the pre-existing PVC. This occurs if the + # application updates outside of application-update. For + # example if the old application is removed and deleted, and the + # new application is uploaded and applied. + convertPVC + requestRekey + } + + # Test whether the specified openbao server(s) agree with the + # specified status of the specified endpoint + # + # Print DEBUG logs when status is non-conforming (the function will + # be used to wait for conformance). + # + # The first parameter is the openbao API endpoint to check status + # of, either /sys/rekey/init or /sys/rekey/verify + # The second parameter is the quoted string of json data returned + # from openbao REST API call. The data should include these fields, + # which are tested for conformance: + # {"nonce": "S", "started": B, "progress": N, + # "verification_required": B} + # + # The other parameters are the servers to test, specified as + # dash-separated IP address output of getOpenbaoPods (XX-XX-XX-XX) + # + # Returns the normal linux success=0, failure!=0 + function assertRekeyStatus { + local endpoint="$1" + local data="$2" + shift 2 + local -a servers=($@) + local -a key_arr + local required + local jscript + local key + local index + local error + local server + local response + local record + + required="nonce progress started verification_required" + jscript=".nonce, .progress, .started, .verification_required" + if [ "$endpoint" == "/sys/rekey/verify" ]; then + required="nonce progress started" + jscript=".nonce, .progress, .started" + fi + + # quick check to assure the data parameter is sane + key_arr=($(echo "$data" | jq -r 'keys[]' | sort)) + for key in $required; do + if [[ " ${key_arr[*]} " != *" $key "* ]]; then + log $ERROR "assertRekeyStatus requires: [$required]," \ + "received: ${key_arr[*]}" + return 1 + fi + done + + required="$( echo "$data" | jq -r "$jscript" )" + + index=0 + error=0 + while [ "$index" -lt "${#servers[@]}" ]; do + server="${servers[$index]}" + index=$((index+1)) + server="${server}.$POD_TARGET_BASE" + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI response GET "$server" "$endpoint" + if [ $? -ne 0 -o -z "$response" ]; then + # failing the REST API call is not the same + # as non-conformance + return 2 + continue + fi + + record="$( echo "$response" | jq -r "$jscript" )" + if [ "$record" != "$required" ]; then + log $ERROR "$server does not conform to:" \ + "$( echo "$data" | jq -c '.' )" + log $DEBUG "$server does not confirm: $response" + error=1 + continue + fi + log $DEBUG "$server conforms: $response" + done + + return $error + } + + # Test whether the openbao server(s) agree about rekey status + # + # The parameter is the quoted string of json data to pass to + # assertRekeyStatus + # + # Returns the normal linux success=0, failure!=0 + function assertServerStatus { + local reference="$1" + local pods + local count + + pods="$( getOpenbaoPods | awk '{print $2}' )" + count="$( echo $pods | wc -w )" + if [ "$count" -ne "$HA_REPLICAS" ]; then + log $ERROR "server without IP does not conform" + return 1 + fi + assertRekeyStatus "/sys/rekey/init" "$reference" $pods + } + + # Test whether the openbao server(s) agree about rekey validation + # status. Warn when the active openbao server changes + # + # The parameter is the quoted string of json data to pass to + # assertRekeyStatus + # + # Returns the normal linux success=0, failure!=0 + function assertVerifyStatus { + local reference="$1" + local response + local pods + local result + local count + + # first assert the rekey status; /sys/rekey/verify returns + # error if a server does not have rekey in progress + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI response GET $ACTIVE_TARGET /sys/rekey/init + result=$? + if [ "$result" -ne 0 ]; then + return $result + fi + assertServerStatus "$response" + result=$? + if [ $result -ne 0 ]; then + return $result + fi + + pods="$( getOpenbaoPods | awk '{print $2}' )" + count="$( echo $pods | wc -w )" + if [ "$count" -ne "$HA_REPLICAS" ]; then + log $ERROR "server without IP does not conform" + return 1 + fi + assertRekeyStatus "/sys/rekey/verify" "$reference" $pods + } + + # Assert that the /sys/rekey/init endpoint reports no + # rekey procedure in progress on any server + # + # Returns the normal linux success=0, failure!=0 + function assertNoRekey { + local data + + data='{"nonce": "", "started": false, "progress": 0' + data="$data"', "verification_required": false}' + assertServerStatus "$data" + } + + # Retrieve the rekey status from active openbao server + # and assert that all server conform to the status + # + # Returns the normal linux success=0, failure!=0 + function assertServersConform { + local response + local value + local result + local pods + local count + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI response GET $ACTIVE_TARGET /sys/rekey/init + if [ $? -ne 0 ]; then + # cannot check conformance + log $ERROR "Cannot check server conformance to" \ + "/sys/rekey/init" + return 2 + fi + + assertServerStatus "$response" + result="$?" + if [ "$result" -ne 0 ]; then + return $result + fi + + value="$( echo "$response" | jq -r '.verification_nonce' )" + if [ -z "$value" -o "$value" == "null" ]; then + return 0 + fi + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI response GET $ACTIVE_TARGET /sys/rekey/verify + if [ $? -ne 0 ]; then + # cannot check conformance + log $ERROR "Cannot check server conformance to" \ + "/sys/rekey/verify" + return 2 + fi + + pods="$( getOpenbaoPods | awk '{print $2}' )" + count="$( echo $pods | wc -w )" + if [ "$count" -ne "$HA_REPLICAS" ]; then + log $ERROR "server without IP does not conform" + return 1 + fi + assertRekeyStatus "/sys/rekey/verify" "$response" $pods + } + + # This function is used during the pre-rekey assertions + # Testing if the main loop (via PODREC_F) indicates a server + # is not running. + function allServersRunning { + local records + local count + + records="$( grep "^/$OPENBAO_FN" "$PODREC_F" )" + count="$( awk -F/ '{print $2}' <<<"$records" | wc -w )" + if [ "$count" -ne "$HA_REPLICAS" ]; then + return 1 + fi + return 0 + } + + # This function is used during the pre-rekey assertions + # Testing if the main loop (via PODREC_F) indicates a server + # is sealed + function allServersUnsealed { + local records + local count + + records="$( grep "^/$OPENBAO_FN" "$PODREC_F" )" + count="$( grep "/false/" <<<"$records" \ + | awk -F/ '{print $2}' | wc -w )" + if [ "$count" -ne "$HA_REPLICAS" ]; then + return 1 + fi + + return 0 + } + + # This function is used during the pre-rekey assertions + # Testing if the main loop (via PODREC_F) indicates a server + # omits IP address + function allServersHaveIP { + local records + local count + + records="$( grep "^/$OPENBAO_FN" "$PODREC_F" )" + count="$( echo "$records" | awk -F/ '{print $3}' | wc -w )" + if [ "$count" -ne "$HA_REPLICAS" ]; then + return 1 + fi + return 0 + } + + # Check the openbao server pods' metadata label "openbao-version", + # and assert that all servers are running the expected version + # which is coded in openbao-manager values.yaml server.version + function allServersCurrent { + local jdata + local podcount + local i + local poddata + local name + local version + + jdata="$( kubectl get pods -n "$OPENBAO_NS" -o json )" + podcount="$( echo "$jdata" | jq ".items | length" )" + + for i in $( seq 0 $((podcount -1 )) ); do + poddata="$( echo "$jdata" | jq ".items[$i]" )" + name="$( echo "$poddata" | jq -r ".metadata.name" )" + if ! [[ "$name" =~ ^${OPENBAO_FN}-[0-9]$ ]]; then + # this is not a openbao server pod + continue + fi + + version="$( echo "$poddata" \ + | jq -r '.metadata.labels["openbao-version"]' )" + if [ "$version" != "$OPENBAO_VERSION" ]; then + log $INFO "Openbao server pod $name is version $version" + return 1 + fi + + log $DEBUG "Openbao server pod $name is version $version" + done + return 0 + } + + # Test the status of rekey procedure 'started' during pre-rekey + # tests for procedure progress selection (sharing a single openbaoAPI + # call to GET /sys/rekey/init + # + # Return linux true (0) if the status of /sys/rekey/init includes + # started == true + # + # Optional argument --not inverts the logic, but maintains + # error response 2 + function assertRekeyStarted { + local started + local not="$1" + + # assert that a rekey is in progress + started="$( echo "$REKEY_STATUS_JSON" | jq -r '.started' )" + if [ "$started" == "true" ]; then + started=0 + elif [ "$started" != "false" ]; then + # the rekey status is unclear + # an error is probably printed + log $DEBUG "unclear response for /sys/rekey/init:" \ + "$( jq -c <<<"$REKEY_STATUS_JSON" )" + return 2 + else + started=1 + fi + + if [ "$started" -eq 0 ]; then + if [ "$not" == "--not" ]; then + return 1 + fi + return 0 + fi + if [ "$not" == "--not" ]; then + return 0 + fi + return 1 + } + + # Delete the shard secrets with speficied prefix + # + # The secrets are deleting on a single kubectl command + function deleteShardSecrets { + local prefix="$1" + local i + local list='' + + for i in $( seq 0 $((KEY_SECRET_SHARES-1)) ); do + if [ -n "$( secretExists "${prefix}-$i" )" ]; then + list="$list ${prefix}-$i" + fi + done + if [ -n "$list" ]; then + deleteSecrets $list + return $? + fi + return 0 + } + + # Make a copy of the shard secrets with specified prefix + # + # The calling function needs to verify the result + function copyShardSecrets { + local from="$1" + local to="$2" + local i + + for i in $( seq 0 $((KEY_SECRET_SHARES-1))); do + get_secret "${from}-$i" \ + | set_secret "${to}-$i" /dev/stdin + if [ $? -ne 0 ]; then + # don't try anything else + log $ERROR "Failed to copy ${from}-$i to ${to}-$i" + break + fi + done + } + + # Just log the content of cluster-rekey-request again + # + # Keeps track of whether openbao-manager has been restarted + # with REKEY_STARTED variable, so that the rekey procedure + # status is documented in log + function rekeyResuming { + if [ "$REKEY_STARTED" -ne 0 ]; then + log $INFO "Resuming rekey:" \ + "$( get_secret cluster-rekey-request )" + REKEY_STARTED=0 + fi + } + + # Return linux true (0) if a rekey is requested and the openbao + # server pods are in a stable condition + # + # If the openbao servers are not "stable" then the rekey operation + # needs that stability first. openbao-manager's main runStateMachine + # will monitor pods and restore unsealed status. + function needsRekey { + local pods + local sealed + local response + local apiversion + + # the first milestone to be created is cluster-rekey-request; + # the last milestone to be deleted is cluster-rekey-audit; + # proceed if any exists + secretsExistAny cluster-rekey-request \ + cluster-rekey-verified \ + cluster-rekey-shuffle \ + cluster-rekey-audit + if [ $? -ne 0 ]; then + # rekey is not requested + return 1 + fi + + # progress the rekey procedure only if the servers are all + # running + if ! allServersRunning; then + log $INFO "Rekey: wait for openbao servers to equal" \ + "$HA_REPLICAS" + return 1 + fi + + # progress the rekey procedure only if the servers were + # previously unsealed. + if ! allServersUnsealed; then + log $INFO "Rekey: wait for unsealed openbao servers to" \ + "equal $HA_REPLICAS" + return 1 + fi + + # progress the rekey procedure only if the servers all have + # DNS names (IP addresses) provided by k8s + if ! allServersHaveIP; then + log $INFO "Rekey: wait for $HA_REPLICAS openbao servers" \ + "to have IP addresses" + return 1 + fi + + # progress a rekey if all server pods are running the expected + # server version + if ! allServersCurrent; then + log $INFO "Rekey: wait for openbao servers to be updated" \ + "to the current version $OPENBAO_VERSION" + return 1 + fi + + # The above four tests are based on output of kubectl get pods + # command. Doublecheck with REST API call to each server + pods="$( getOpenbaoPods | grep "^$OPENBAO_FN" | awk '{print $2}' )" + for pod in $pods; do + NO_HEADER=true \ + API_TMOUT=$QUERY_TMOUT \ + openbaoAPI response GET ${pod}.$POD_TARGET_BASE /sys/health + if [ $? -ne 0 ]; then + log $ERROR "$pod fails health check during rekey" + return 1 + fi + sealed="$( echo "$response" | jq -r '.sealed' )" + if [ "$sealed" != "false" ]; then + log $ERROR "$pod is sealed during rekey" + return 1 + fi + apiversion="$( echo "$response" | jq -r '.version' )" + if [ "$apiversion" != "$OPENBAO_VERSION" ]; then + log $ERROR "$pod is not version $OPENBAO_VERSION" + return 1 + fi + done + + assertServersConform + return $? + } + + # Return linux true (0) if the current step of the rekey procedure + # is to send initialize request to /sys/rekey/int + # + # Initialize is the first step + # + # Will not begin initialization if there are stale cluster-rekey or + # cluster-key-bk secrets + function needsInitialization { + local progress + local count + local error=0 + + assertRekeyStarted --not + progress=$? + if [ "$progress" -ne 0 ]; then + return "$progress" + fi + + # skip if this represents a recovery path + secretsExistAny cluster-rekey-verified \ + cluster-rekey-shuffle \ + cluster-rekey-audit + if [ $? -eq 0 ]; then + return 1 + fi + + # make assertions about the artifacts left behind by previous + # rekey procedure attempts + # assert that there are no stale keys before starting rekey + assertShardSecrets cluster-rekey --nokeys + count=$? + if [ "$count" -ne 0 ]; then + log $ERROR "Stale cluster-rekey secrets ($count) present" + # there was a possibility that openbao had cancelled the rekey + # due to active server failure, so fall through to + # rekeyRecovery + return 1 + fi + + assertShardSecrets cluster-key-bk --nokeys + count=$? + if [ "$count" -ne 0 ]; then + log $ERROR "cluster-key-bk secrets ($count) present" + return 2 + fi + + return 0 + } + + # Start the rekey procedure + # + # Send initialize request to /sys/rekey/int + # + # Initialize is the first step + # + # Will not begin initialization if there are stale cluster-rekey or + # cluster-key-bk secrets + function rekeyInitialize { + local shares + local threshold + local verify + local data + local response + local value + + log $INFO "Initializing openbao rekey" + + REKEY_STARTED=0 + + shares='"secret_shares": '$KEY_SECRET_SHARES + threshold='"secret_threshold": '$KEY_REQUIRED_THRESHOLD + verify='"require_verification": true' + data="{$shares,$threshold,$verify}" + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response POST $ACTIVE_TARGET /sys/rekey/init "$data" + if [ $? -ne 0 ]; then + return 1 + fi + + value="$( echo "$response" | jq -r ".started" )" + if [ 'false' == "$value" ]; then + log $ERROR "Rekey not started" + return 1 + fi + + # log the nonce + value="$( echo "$response" | jq -r ".nonce" )" + verify="$( echo "$response" | jq -r ".verification_required" )" + log $INFO "Rekey started: $value" \ + "(verification_required==$verify)" + + # just a sanity check + if [ 'true' != "$verify" ]; then + log $ERROR "Rekey started without verification_required:" \ + "aborting" + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response DELETE $ACTIVE_TARGET /sys/rekey/init + return 1 + fi + + assertServerStatus "$response" + return $? + } + + # The rekey authentication should happen when + # - there is a rekey in progress + # - there is a verification_nonce + # + # Authentication of the rekey request is the second step + # + # Omit rekey verification if: + # - there are existing cluster-rekey secrets + # - Verification is complete: cluster-rekey-verified or any later + # stage is complete + # + # Return linux true (0) if the current stage of rekey + # is to complete the rekey verification + # Return linux true (0) if the current stage of rekey + # is to authentication the rekey request + function needsAuthentication { + local progress + + assertRekeyStarted + progress=$? + if [ "$progress" -ne 0 ]; then + return "$progress" + fi + + progress="$( echo "$REKEY_STATUS_JSON" \ + | jq -r '.verification_nonce' )" + if ! [ -z "$progress" -o "$progress" == "null" ]; then + # There is a rekey in progress with a verification nonce + # pass through to recovery + return 1 + fi + + # this represents a recovery path + assertShardSecrets cluster-rekey --nokeys + if [ $? -ne 0 ]; then + # There are already cluster-rekey secrets + return 1 + fi + + # skip if this represents a recovery path + secretsExistAny cluster-rekey-verified \ + cluster-rekey-shuffle \ + cluster-rekey-audit + if [ $? -eq 0 ]; then + return 1 + fi + + return $? + } + + # Submits a keyshard for the rekey procedure + # Returns 0 on success + # Returns 1 on failure + # Returns KEY_SECRET_SHARES when authentication completes + function rekeySubmitShard { + local nonce="$1" + local index="$2" + local verifyauth="$3" + local prefix="$4" + local shard + local dnonce + local key + local data + local response + local progress + local root_token + local new_doc + + if [ -z "$prefix" ]; then + prefix=cluster-key + fi + + shard="$( get_secret "${prefix}-$index" | jq -r .keys[0] )" + dnonce='"nonce": "'$nonce'"' + key='"key": "'$shard'"' + data="{$dnonce,$key}" + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response POST $ACTIVE_TARGET /sys/rekey/update "$data" + if [ $? -ne 0 ]; then + return 1 + fi + + # Check the response for verification_nonce, which + # indicates completion + progress="$( echo "$response" | jq -r '.verification_nonce' )" + if [ -n "$progress" -a "$progress" != 'null' ]; then + log $INFO "Success authenticating:" \ + "$((index+1)) of $KEY_REQUIRED_THRESHOLD" + + if [ "$verifyauth" == "--verify-auth" ]; then + # delete the rekey and return success + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response DELETE $ACTIVE_TARGET /sys/rekey/init + return "$KEY_SECRET_SHARES" + fi + + # Procedure to ensure that the old and new shards are + # secured in k8s secrets. Deletion of old shards will only + # occur when verification is successful. + root_token="$( get_secret cluster-key-root )" + new_doc="$( echo "$response" \ + | jq -c '{"keys": .keys, + "keys_base64": .keys_base64, + "root_token": "'"$root_token"'"}' )" + # store the new shards + echo "$response" \ + | jq -c '{"keys": .keys, "keys_base64": .keys_base64}' \ + | storeOpenbaoInitSecrets cluster-rekey + + # check that the secrets match openbao's rekey response + echo "$new_doc" | validateSecrets cluster-rekey + if [ $? -ne 0 ]; then + # calling function will abort the rekey + # and any cluster-rekey secrets + log $ERROR "Failed to store and verify shards" \ + "after rekey authentication complete" + return 1 + fi + + # authentication of the rekey request is completed + # successfully + log $INFO "Rekey authentication successful" + return "$KEY_SECRET_SHARES" + fi + + # Otherwise verify the response + progress="$( echo "$response" | jq -r '.progress' )" + index="$((index+1))" + if [ "$progress" -ne "$index" ]; then + log $ERROR "Authentication sequence mismatching" \ + "($progress, $index)" + return 1 + fi + + # assert that the servers agree + assertServerStatus "$response" + if [ $? -ne 0 ]; then + log $ERROR "Openbao server rekey status fails during" \ + "authentication at $index of $KEY_REQUIRED_THRESHOLD" + return 1 + fi + + log $INFO "Success authenticating:" \ + "$index of $KEY_REQUIRED_THRESHOLD" + return 0 + } + + # Return linux true (0) if the current step of the rekey procedure + # is to authenticate the request + # + # Authentication of the rekey request is the second step + # + function rekeyAuthenticate { + local verifyauth="$1" + local prefix="$2" + local response + local index + local value + local nonce + local progress + local result + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI response GET $ACTIVE_TARGET /sys/rekey/init + if [ $? -ne 0 ]; then + # an error is already printed + return 1 + fi + + value="$( echo "$response" | jq -r '.started' )" + if [ 'true' != "$value" ]; then + log $ERROR "Rekey authentication, but rekey not in progress" + return 1 + fi + + nonce="$( echo "$response" | jq -r '.nonce' )" + progress="$( echo "$response" | jq -r '.progress' )" + if ! [[ "$progress" =~ ^[0-9]{1,}$ ]]; then + log $ERROR "Rekey authentication progress not integer:" \ + "$response" + return 1 + elif [ "$progress" -ge "$KEY_SECRET_SHARES" ]; then + log $ERROR "Rekey authentication progress out of range:" \ + "$response" + return 1 + fi + + if [ "$progress" -ne 0 ]; then + log $WARNING "Continue authenticating rekey at: $progress" + fi + + # authenticate and store the new keys + for index in $( seq $progress $((KEY_SECRET_SHARES-1)) ); do + rekeySubmitShard "$nonce" "$index" $verifyauth $prefix + result="$?" + if [ "$result" -eq "$KEY_SECRET_SHARES" ]; then + # start the verify procedure now + if [ "$verifyauth" != "--verify-auth" ]; then + log $INFO "Starting rekey verify" + fi + break + elif [ "$result" -ne 0 ]; then + return $result + fi + done + return 0 + } + + # The rekey verification should happen when + # - there is a rekey in progress + # - there is a verification_nonce + # + # Omit rekey verification if: + # - there are existing cluster-rekey secrets + # - Verification is complete: cluster-rekey-verified or any later + # stage is complete + # + # Return linux true (0) if the current stage of rekey + # is to complete the rekey verification + function needsVerify { + local progress + + assertRekeyStarted + progress=$? + if [ "$progress" -ne 0 ]; then + return "$progress" + fi + + progress="$( echo "$REKEY_STATUS_JSON" \ + | jq -r '.verification_nonce' )" + if [ -z "$progress" -o "$progress" == "null" ]; then + # There is a rekey in progress, but not with a + # verification nonce + return 1 + fi + + # Assert that the nonce is UUID-ish + if ! [[ "$progress" =~ ^[a-f0-9-]{36}$ ]]; then + log $ERROR "The verification_nonce is not UUID-ish:" \ + "$REKEY_STATUS_JSON" + return 2 + fi + + assertShardSecrets cluster-rekey + if [ $? -ne 0 ]; then + # this should not happen: verify in progress but no + # cluster-rekey secrets + log $ERROR "rekey verify in progress but no cluster-rekey" + return 1 + fi + + # skip if this represents a recovery path + secretsExistAny cluster-rekey-verified \ + cluster-rekey-shuffle \ + cluster-rekey-audit + if [ $? -eq 0 ]; then + return 1 + fi + + return 0 + } + + # Submits a keyshard for the rekey verification procedure + # Returns 0 on success + # Returns 1 on failure + # Returns KEY_REQUIRED_THRESHOLD when authentication completes + function rekeyVerifySubmitShard { + local nonce="$1" + local index="$2" + local shard + local dnonce + local key + local data + local response + local progress + + shard="$( get_secret cluster-rekey-$index \ + | jq -r .keys[0] )" + dnonce='"nonce": "'$nonce'"' + key='"key": "'$shard'"' + data="{$dnonce,$key}" + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response POST $ACTIVE_TARGET \ + /sys/rekey/verify "$data" + if [ $? -ne 0 ]; then + # an error is printed + return 1 + fi + + progress="$( echo "$response" | jq -r ".complete" )" + if [ "$progress" == 'true' ]; then + log $INFO "Success verifying: using new shards" + set_secret cluster-rekey-verified /dev/stdin \ + <<<"$( get_secret cluster-rekey-request )" + return $KEY_REQUIRED_THRESHOLD + fi + progress="$( echo "$response" | jq -r ".progress" )" + if [ -z "$progress" -o "$progress" == "null" ]; then + log $ERROR "Expecting rekey verify progress" \ + "[$((index+1))] instead of [$progress]" + return 1 + fi + # Print the progress of rekey verify. + if [ "$((index+1))" -eq "$progress" ]; then + log $INFO "Success verifying:" \ + "$progress of $KEY_REQUIRED_THRESHOLD" + elif [ "$((index+1))" -gt "$progress" ]; then + # A sanity check only + log $WARNING "Verify progress [$progress] less" \ + "than expected [$((index+1))]" + else + # A sanity check only + log $WARNING "Verify progress [$progress]" \ + "greater than expected [$((index+1))]" + fi + assertVerifyStatus "$response" + if [ $? -ne 0 ]; then + log $ERROR "Openbao server verify status fails during" \ + "authentication at" \ + "$index of $KEY_REQUIRED_THRESHOLD" + return 1 + fi + } + + # Return linux true (0) if the current step of the rekey procedure + # is to verify shard secrets + # + # This step confirms that openbao manager has correctly stored the + # shards received from the openbao server. This allows failures of + # the procedure to be recovered: + # - receive the shards from openbao + # - store the shards in k8s secrets + # - play the shards back to openbao + # - upon successful verification the new shards are effective + # + # Verification of the rekey request is the Third step + # + function rekeyVerify { + local value + local nonce + local progress + local response + local shard + local dnonce + local key + local data + local index + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI response GET $ACTIVE_TARGET /sys/rekey/verify + if [ $? -ne 0 ]; then + # an error is already printed + return 1 + fi + + value="$( echo "$response" | jq -r '.started' )" + if [ 'true' != "$value" ]; then + log $ERROR "Rekey verify, but rekey not in progress" + return 1 + fi + + nonce="$( echo "$response" | jq -r '.nonce' )" + progress="$( echo "$response" | jq -r '.progress' )" + if ! [[ "$progress" =~ ^[0-9]{1,}$ ]]; then + log $ERROR "Rekey authentication progress not integer:" \ + "$response" + return 1 + elif [ "$progress" -ge "$KEY_SECRET_SHARES" ]; then + log $ERROR "Rekey authentication progress out of range:" \ + "$response" + return 1 + fi + if [ "$progress" -ne 0 ]; then + log $WARNING "Continue verifying rekey at: $progress" + fi + + # assert that the servers agree on verify status + assertVerifyStatus "$response" + if [ $? -ne 0 ]; then + return 1 + fi + + # authenticate the verify procedure + for index in $( seq $progress $((KEY_SECRET_SHARES-1)) ); do + rekeyVerifySubmitShard "$nonce" "$index" + result=$? + if [ "$result" -eq "$KEY_REQUIRED_THRESHOLD" ]; then + # rekeyVerifySubmitShard returns KEY_REQUIRED_THRESHOLD + # when .complete == true was received + return 0 + elif [ "$result" -ne 0 ]; then + # any other non-zero result is a failure + return 1 + fi + done + + log $ERROR "Verify procedure ended without completion" + return 1 + } + + # The shuffling of keys shards in k8s secrets should happen when + # th cluster-rekey-verified procedure step is completed. + # + # Omit shuffling if: + # - openbao server reports rekey in progress (unclear status) + # - shuffling is already complete: cluster-rekey-shuffle or later + # stage is complete + # - there are no cluster-rekey secrets + # - there are cluster-key-bk secrets + # + # Return linux true (0) if the current stage of rekey + # is to complete the swapping of validated shards + function needsShuffle { + local progress + + # assert that a rekey is not in progress + assertRekeyStarted --not + progress=$? + if [ "$progress" -ne 0 ]; then + # 1 - maintain the status of rekey in progress + # 2 - api error, try again later + return "$progress" + fi + + secretExists cluster-rekey-verified >/dev/null + if [ $? -ne 0 ]; then + # proceeds to next procedure step + return 1 + fi + + # skip if this represents a recovery path + secretsExistAny cluster-rekey-shuffle \ + cluster-rekey-audit + if [ $? -eq 0 ]; then + return 1 + fi + + assertShardSecrets cluster-rekey + case $? in + 0) + # There is no rekey in progress, and there is a set + # of cluster-rekey shards recorded + ;; + $KEY_SECRET_SHARES) + # There is no rekey in progress, and there are no + # cluster-rekey shards recorded + return 1 + ;; + *) + # with cluster-rekey-verified, an incomplete set of + # cluster-rekey indicates partial deletion after copying + # to cluster-key + # will want to audit the cluster-key secrets before + # deleting cluster-rekey + log $WARNING "The number key shard secrets for" \ + "cluster-rekey is not complete" + return 1 + ;; + esac + + # otherwise allow rekeyShuffleKeys to be re-entrant to + # the existance of or lack of cluster-key and cluster-key-bk + # cluster-rekey is only deleted when confirmed to be copied to + # cluster-key + return 0 + } + + # This procedure shuffles the shard secrets from cluster-rekey to + # cluster-key to cluster-bk + # + # The function intends to be resolve failures of the openbao manager + # process where it is interrupted abruptly such as with kill -9. + # In combination with needsShuffle it can be re-run until it + # completes the shuffle: + # - cluster-key shards are copied to cluster-key-bk + # - cluster-key shards are delete + # - cluster-rekey is copied to cluster-key + # - cluster-rekey is delete + # + # A subsequent step audits the new keys before deleting the + # cluster-key-bk secrets + function rekeyShuffleKeys { + local key_exists + local rekey_exists + local bk_exists + local key_doc="" + local rekey_doc="" + + assertShardSecrets cluster-key + key_exists=$? + assertShardSecrets cluster-rekey + rekey_exists=$? + assertShardSecrets cluster-key-bk + bk_exists=$? + + if [ "$key_exists" -eq 0 ]; then + key_doc="$( reconstructInitResponse cluster-key )" + echo "$key_doc" | validateSecrets cluster-key + if [ $? -ne 0 ]; then + log $ERRROR "Failed to read cluster-key" + return 1 + fi + fi + + if [ "$rekey_exists" -eq 0 ]; then + rekey_doc="$( reconstructInitResponse cluster-rekey )" + echo "$rekey_doc" | validateSecrets cluster-rekey + if [ $? -ne 0 ]; then + log $ERROR "Failed to read cluster-rekey" + return 1 + fi + else + # this is recovery path + if [ -n "key_doc" ]; then + log $WARNING "Progress cluster-rekey-shuffle without" \ + "cluster-rekey" + set_secret cluster-rekey-shuffle /dev/stdin \ + <<<"$( get_secret cluster-rekey-request )" + return + fi + log $ERROR "No cluster-key or cluster-rekey" + return 1 + fi + + if [ "$bk_exists" -lt "$KEY_SECRET_SHARES" \ + -a "$bk_exists" -ne 0 ]; then + # this is a recovery path + # an incomplete copy of cluster-key secrets + if [ -n "$key_doc" ]; then + deleteShardSecrets cluster-key-bk + assertShardSecrets cluster-key-bk + bk_exists=$? + if [ "$bk_exists" -lt "$KEY_SECRET_SHARES" ]; then + log $ERROR "Failed to delete incomplete" \ + "cluster-key-bk" + return 1 + fi + else + # this shouldn't happen; + # either not both failures is anticipated + log $ERROR "Sanity: incomplete both cluster-key-bk" \ + "and missing/incomplete cluster-key secrets" + return 1 + fi + fi + if [ "$bk_exists" -eq 0 ]; then + # this is a recovery path + if [ -n "$key_doc" ]; then + # Assert that cluster-key and cluster-key-bk are the + # same + log $INFO "Recovering from pre-existing cluster-key-bk" + echo "$key_doc" | validateSecrets cluster-key-bk + if [ $? -eq 0 ]; then + # cluster-key-bk == cluster-key + deleteShardSecrets cluster-key + assertShardSecrets cluster-key + key_exists=$? + key_doc="" + else + echo "$key_doc" | validateSecrets cluster-rekey + if [ $? -eq 0 ]; then + # Recovering cluster-key == cluster-rekey + log $INFO "Recovering with cluster-key" + deleteShardSecrets cluster-rekey + set_secret cluster-rekey-shuffle /dev/stdin \ + <<<"$( get_secret cluster-rekey-request )" + return 0 + else + log $ERROR "Three different sets of keys" \ + "in k8s secrets" + return 1 + fi + fi + fi + # else: there is no cluster-key to backup + else + # this is the normal procedure path + log $INFO "Copying cluster-key secrets to cluster-key-bk" + copyShardSecrets cluster-key cluster-key-bk + echo "$key_doc" | validateSecrets cluster-key-bk + if [ $? -ne 0 ]; then + log $ERROR "Failed to copy cluster-key to cluster-key-bk" + deleteShardSecrets cluster-key-bk + return 1 + fi + deleteShardSecrets cluster-key + if [ $? -ne 0 ]; then + log $ERROR "Failed to delete cluster-key secrets" + return 1 + fi + assertShardSecrets cluster-key + key_exists=$? + key_doc="" + fi + + # cluster-key-bk exists here + # cluster-rekey rekey_doc is valid here + + # if cluster-key exists, such as number of secrets less than + # KEY_SECRET_SHARES, then delete them; deleteShardSecrets is a + # no-op if there are none there + deleteShardSecrets cluster-key + if [ $? -ne 0 ]; then + log $ERROR "Failed to delete cluster-key" + return 1 + # try again later + fi + + log $INFO "Copying cluster-rekey secrets to cluster-key" + copyShardSecrets cluster-rekey cluster-key + echo "$rekey_doc" | validateSecrets cluster-key + if [ $? -ne 0 ]; then + log $ERROR "Failed to copy cluster-rekey to cluster-key" + return 1 + fi + + deleteShardSecrets cluster-rekey + set_secret cluster-rekey-shuffle /dev/stdin \ + <<<"$( get_secret cluster-rekey-request )" + + return 0 + } + + # The audit of cluster-key should happen when these other procedure + # steps are completed: + # - cluster-rekey-verified + # - cluster-rekey-shuffle + # + # Omit audit if: + # - openbao server reports rekey in progress (failed previous audit?) + # - audit is already complete: cluster-rekey-audit exists + # + # Return linux true (0) if the current stage of rekey + # is to run the audit + function needsAudit { + local progress + + # assert that a rekey is not in progress + assertRekeyStarted --not + progress=$? + if [ "$progress" -ne 0 ]; then + return "$progress" + fi + + # Select recovery path with response '3' + secretExists cluster-rekey-audit >/dev/null + if [ $? -eq 0 ]; then + # this path indicates a failure to complete + # finalizeRekey. cluster-rekey-audit is the last + # milestone to be deleted + log $INFO "rekey audit already completed" + return 3 + fi + + secretExists cluster-rekey-request >/dev/null + if [ $? -ne 0 ]; then + return 1 + fi + + secretExists cluster-rekey-verified >/dev/null + if [ $? -ne 0 ]; then + return 1 + fi + + secretExists cluster-rekey-shuffle >/dev/null + if [ $? -ne 0 ]; then + return 1 + fi + + assertShardSecrets cluster-key + if [ $? -ne 0 ]; then + log $ERROR "rekey audit requested but cluster-keys absent" + return 1 + fi + } + + # Audit that the active openbao server authenticates with the cluster + # keys specified by prefix + # + # Returns 0 on success + # Returns 1 if the audit failes + # Returns 2 if there was a failure unrelated to authentication + function rekeyAudit { + local prefix="$1" + local value + local response + + if [ -z "$prefix" ]; then + prefix="cluster-key" + fi + + log $INFO "Auditing the shards in $prefix secrets" + assertNoRekey + if [ $? -ne 0 ]; then + log $ERROR "Cannot audit with rekey in progress" + return 2 + fi + + assertShardSecrets "$prefix" + if [ $? -ne 0 ]; then + log $ERROR "Audit fails with absent $prefix secrets" + return 1 + fi + + rekeyInitialize + if [ $? -ne 0 ]; then + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response DELETE $ACTIVE_TARGET /sys/rekey/init + return 2 + fi + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI response GET $ACTIVE_TARGET /sys/rekey/init + if [ $? -ne 0 ]; then + # There's no reason to believe this one will succeed where + # the other hadn't + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response DELETE $ACTIVE_TARGET /sys/rekey/init + return 2 + fi + + value="$( echo "$response" | jq -r ".verification_required" )" + if [ "$value" != "true" ]; then + log $ERROR "Audit sanity: verification_required not set:" \ + "$response" + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response DELETE $ACTIVE_TARGET /sys/rekey/init + return 1 + fi + + rekeyAuthenticate --verify-auth "$prefix" + result="$?" + if [ "$result" -eq 0 ]; then + log $INFO "Audit of cluster-key secrets passes" + else + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response DELETE $ACTIVE_TARGET /sys/rekey/init + fi + + return $result + } + + # clean up the artifacts from rekey procedure + # The audit procedure proves the shards in cluster-key + # secrets will unseal the openbao. + # + # If openbao-manager is killed during this procedure step it should + # continue to try to delete the artifacts until finally deleting + # cluster-rekey-audit + function finalizeRekey { + local secrettext + secrettext="$( get_secret cluster-rekey-audit )" + + log $INFO "removing artifacts of the rekey procedure:" \ + "$secrettext" + assertShardSecrets cluster-rekey --nokeys + if [ $? -ne 0 ]; then + log $WARNING "removing cluster-rekey secrets" \ + "after audit" + deleteShardSecrets cluster-rekey + fi + deleteShardSecrets cluster-key-bk + deleteSecrets cluster-rekey-verified + deleteSecrets cluster-rekey-shuffle + deleteSecrets cluster-rekey-request + deleteSecrets cluster-rekey-audit + + log $INFO "Rekey request complete: $secrettext" + } + + # This procedure handle a few cases where the openbao active server or + # openbao-manager were killed. + # + # - rekey authentication completed by openbao-manager was killed + # before the shards could be stored + # - rekey verification may be cancelled by the failure of the active + # openbao server + # + function rekeyRecovery { + local key_exists + local rekey_exists + local bk_exists + local verified_exists + local shuffle_exists + local audit_exists + local inprogress + local verifyprogress + + log $INFO "Recovering the rekey procedure" + + # assert that the openbao server are all up and agree + # about the rekey status + allServersRunning \ + && allServersHaveIP \ + && allServersUnsealed \ + || return 1 + + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI REKEY_STATUS_JSON GET $ACTIVE_TARGET /sys/rekey/init + if [ $? -ne 0 ]; then + # an error is printed + # wait for recovery + REKEY_STATUS_JSON='' + return 1 + fi + assertServerStatus "$REKEY_STATUS_JSON" + if [ $? -ne 0 ]; then + # wait for the openbao servers to sync + return 1 + fi + + inprogress="$( echo "$REKEY_STATUS_JSON" | jq -r '.started' )" + verifyprogress="$( echo "$REKEY_STATUS_JSON" \ + | jq -r '.verification_nonce' )" + if [ "$inprogress" == "true" ]; then + # If a rekey is in progress, then cancel it + # - an authentication will reinitialize + # - a verification will reinitialtize + # - a rekeyAudit will retry + log $INFO "Cancelling rekey in progress" + NO_HEADER=true \ + API_TMOUT=$API_REKEY_OP_TMOUT \ + openbaoAPI response DELETE $ACTIVE_TARGET /sys/rekey/init + if [ $? -ne 0 ]; then + # retry later + return 1 + fi + fi + + assertShardSecrets cluster-key + key_exists=$? + assertShardSecrets cluster-rekey + rekey_exists=$? + assertShardSecrets cluster-key-bk + bk_exists=$? + + secretExists cluster-rekey-verified >/dev/null + verified_exists=$? + secretExists cluster-rekey-shuffle >/dev/null + shuffle_exists=$? + secretExists cluster-rekey-audit >/dev/null + audit_exists=$? + + # review each of the milestones to discern the failure point + if [ "$audit_exists" -eq 0 ]; then + true + # no recovery options here + # pass through + elif [ "$shuffle_exists" -eq 0 ]; then + true + # no recovery options here + # pass through + elif [ "$verified_exists" -eq 0 ]; then + if [ "$rekey_exists" -gt 0 ]; then + if [ "$rekey_exists" -lt "$KEY_SECRET_SHARES" ]; then + # with verified_exists, indicates partial deletion + # of the cluster-rekey secrets after copying to + # cluster-key. Audit the cluster-key secrets before + # deleting rekey + rekeyAudit cluster-key + if [ $? -ne 0 ]; then + log $ERROR "Audit cluster-key fails with a" \ + "partial set of cluster-rekey" + return 1 + fi + + deleteShardSecrets cluster-rekey + fi + + # Handle condition where secrets were shuffled but + # openbao-manager failed before recording the + # milestone cluster-rekey-shuffle + + # auditRekey will double-check that cluster-key is + # in use + set_secret cluster-rekey-shuffle /dev/stdin \ + <<<"$( get_secret cluster-rekey-request )" + log $INFO "Continuing rekey procedure with audit" \ + "of cluster-key" + return 0 + fi + # else: pass through + else + if [ "$rekey_exists" -eq 0 ]; then + # Handle condition where an active server fails during + # verification: openbao may have cancelled the rekey procedure + + # This question is: which shards are the openbao servers + # using? + log $INFO "Recovering from mismatch of cluster-rekey" \ + "and verified status" + + # Audit the existing shards to see which ones the + # openbao servers are keyed for. + # Most likely that the verification failed due to + # active server failing, start with cluster-key + rekeyAudit cluster-key + if [ $? -eq 0 ]; then + # The rekey verification did not complete + # remove cluster-rekey secrets + # The rekey procedure should restart + deleteShardSecrets cluster-rekey + log $INFO "Restart rekey procedure" + return 0 + fi + + # this happens when openbao-manager process is killed + rekeyAudit cluster-rekey + if [ $? -eq 0 ]; then + set_secret cluster-rekey-verified /dev/null \ + <<<$( get_secret cluster-rekey-request ) + log $INFO "Continue rekey procedure with cluster-rekey" + return 0 + fi + # else: pass through + elif [ "$rekey_exists" -eq 5 ]; then + # There are no cluster-rekey secrets; and the rekey is + # cancelled: the rekey procedure will restart + log $INFO "Continue rekey procedure with initialization" + return 0 + else # cluster-rekey secrets are incomplete + # Handle condition where verification is needed but + # openbao-manager did not store shards. The rekey was + # canceled above + + # assert cluster-key before deleteing rekey + rekeyAudit cluster-key + if [ $? -eq 0 ]; then + # the rekey procedure will restart + log $INFO "Deleting partial set of" \ + "cluster-rekey secrets" + deleteShardSecrets cluster-rekey + return 0 + fi + # else: pass through + fi + fi + + log $ERROR "Did not recover from current rekey status" + } + + # The state machine for rekeying the openbao server + # + # The overall procedure for rekey request includes: + # - wait for stability of openbao servers + # - initialize the procedure + # - authenticate the rekey procedure by supplying shards + # - store the new shards + # - verify the rekey with the new shards read from k8s secrets + # - rotate the shard secrets: + # cluster-rekey - cluster-key - cluster-key-bk + # - Audit the new shards with active openbao server + # - Remove artifacts of rekey procedure: + # cluster-key-bk, milestone secrets + # + function openbaoRekey { + local records + local count + local result + local secrettext + + if ! needsRekey; then + return + fi + + # Retrieve and record the rekey status once for the tests that + # follow + NO_HEADER=true \ + API_TMOUT=$API_REKEY_QUERY_TMOUT \ + openbaoAPI REKEY_STATUS_JSON GET $ACTIVE_TARGET /sys/rekey/init + if [ $? -ne 0 ]; then + # an error is printed + REKEY_STATUS_JSON='' + return + fi + + needsAudit + case $? in + 0) + rekeyResuming + rekeyAudit + if [ $? -eq 0 ]; then + set_secret cluster-rekey-audit /dev/stdin \ + <<<$( get_secret cluster-rekey-request ) + + finalizeRekey + fi + return + ;; + 1) # continue to procedure step + ;; + 3) # audit is already completed + secretExists cluster-rekey-audit >/dev/null + if [ $? -eq 0 ]; then + # the cluster-key secrets were audit, but openbao + # manager didn't get a chance to set + # cluster-rekey-audit milestone + finalizeRekey + return + fi + log $ERROR "Discrepancy between needsAudit and" \ + "rekeyOpenbao" + return + ;; + *) + # an error occurs for which the procedure should not + # continue + return + ;; + esac + + needsShuffle + case $? in + 0) + rekeyResuming + rekeyShuffleKeys + return + ;; + 1) # continue to procedure step + ;; + *) + # an error occurs for which the procedure should not + # continue + return + ;; + esac + + needsVerify + case $? in + 0) + rekeyResuming + rekeyVerify + return + ;; + 1) # continue to procedure step + ;; + *) + # an error occurs for which the procedure should not + # continue + return + ;; + esac + + needsAuthentication + case $? in + 0) + rekeyResuming + rekeyAuthenticate + return + ;; + 1) # continue to procedure step + ;; + *) + # an error occurs for which the procedure should not + # continue + return + ;; + esac + + needsInitialization + case $? in + 0) + secrettext="$( get_secret cluster-rekey-request )" + log $INFO "Rekey request started: $secrettext" + rekeyInitialize + return + ;; + 1) # continue to failure + ;; + *) + # an error occurs for which the procedure should not + # continue + return + ;; + esac + + # falling through the case statements requires remediation + rekeyResuming + rekeyRecovery + } + + # Return 0 (true) if either the openbao server status shows a rekey + # is in progress, or if openbao-manager is engaged in the process of + # rekeying the openbao + # + # Openbao manager rekey is in progress if either of these secrets + # exists: + # cluster-rekey-request - the first to be created + # cluster-rekey-audit - the last to be removed + function rekeyInProgress { + # query the openbao server + assertNoRekey + if [ $? -ne 0 ]; then + return 0 + fi + + # look for openbao-manager's milestone secrets + secretsExistAny cluster-rekey-request cluster-rekey-audit + return $? + } + + # Check conditions that need to be met before taking a snapshot of + # the openbao. The same conditions apply for snapshot restore. + # + # The required conditions are: + # - openbao server pods matches HA_REPLICAS + # - openbao server pods are unsealed + # - there is no rekey in progress + # + # Returns 0 for success, or >0 for conditions not met + # The fail conditions are logged to stdout/stderr + function snapshotPreCheck { + local errors=0 + local pods + local podcount + local host + local dnsname + local server_status + local sealed + + pods="$( getOpenbaoPods | grep "^$OPENBAO_FN" )" + podcount="$( echo "$pods" | awk '{print $1}' | wc -w )" + + if [ "$podcount" -ne "$HA_REPLICAS" ]; then + log $ERROR "snapshotPreCheck: openbao pods ($podcount)" \ + "does not match replicas ($HA_REPLICAS)" + errors=$(( errors + 1 )) + fi + + while read host dnsname; do + NO_HEADER=true \ + API_TMOUT=$QUERY_TMOUT \ + openbaoAPI server_status GET $dnsname.$POD_TARGET_BASE \ + /sys/health + sealed="$( echo "$server_status" | jq .sealed )" + if [ "$sealed" != "false" ]; then + log $ERROR "snapshotPreCheck: $host ($dnsname)" \ + "sealed status is [$sealed]" + errors=$(( errors + 1 )) + else + log $DEBUG "snapshotPreCheck: $host ($dnsname)" \ + "sealed status is [$sealed]" + fi + done <<<"$pods" + + if rekeyInProgress; then + log $ERROR "snapshotPreCheck: a rekey is in progress" + errors=$(( errors + 1 )) + fi + + return $errors + } + + # Take a snapshot of the openbao, which is output to stdout + function snapshotCreate { + local apipath=/sys/storage/raft/snapshot + + curl -s -S --cacert "$CERT" \ + --connect-timeout $QUERY_TMOUT \ + --header "X-Vault-Token:$( get_secret cluster-key-root )" \ + --request "GET" \ + "https://$ACTIVE_TARGET:${TARGET_PORT}/v1${apipath}" + } + + # Store the init response and metadata associated with a openbao + # snapshot into the specified k8s secret. + # + # metadata should be a dictionary type structure in this form: + # {"date":"xxx","snapshot_sum":"yyy","secret":"zzz"} + # + # The 'snapshot' of the init response should be taken promptly with + # the snapshot of the openbao. Especially, consider pausing openbao + # manager, in addition to using snapshotPreCheck, to ensure the + # two are consistent. + # + # In practice the metadata can contain any information; the + # procedure only requires the value of 'secret', as in: + # echo "$metadata" | jq -r .secret + function snapshotSetSecret { + local secret="$1" + local metadata="$2" + local jqlog + local result + local keys + local data + + # make sure the user supplied data is ok + jqlog="$( echo "$metadata" | jq . 2>&1 >/dev/null )" + result=$? + if [ $result -ne 0 ]; then + log $ERROR "snapshotSetSecret: error parsing metadata:" \ + "[$result] [$jqlog]" + return 1 + fi + + # check that the user supplied metadata contains 'secret', + # which is the only value the procedure requires. + jqlog="$( echo "$metadata" | jq -r .secret 2>&1 )" + if [ $? -ne 0 -o -z "$jqlog" -o "$jqlog" == "null" ]; then + log $WARNING "snapshotSetSecret: metadata omits 'secret'" + fi + + keys="$( reconstructInitResponse cluster-key )" + data="{\"metadata\":$metadata,\"init\":$keys}" + + # make sure the assembled secret data is ok + echo "$data" | jq . >/dev/null 2>&1 + result=$? + if [ $result -ne 0 ]; then + log $ERROR "snapshotSetSecret: error parsing secret data:" \ + "[$result]" + return 1 + fi + + echo "$data" | jq -c . | set_secret "$secret" /dev/stdin + + # verify the copy of shards secrets + get_secret "$secret" | jq -c .init | validateSecrets cluster-key + if [ $? -ne 0 ]; then + return 1 + fi + + return 0 + } + + # POST stdin to the active openbao server API endpoint for restoring + # the snapshot. stdin is the snapshot file of the openbao cluster. + # + # The required parameter is the metadata associated with the + # snapshot, which contains the name of the k8s secret which has + # the unseal shards for the openbao data being restored. The metadata + # needs to contain at least '{"secret":"xxx"}', and this secret + # needs to exist in the openbao namespace. + # + # The content of the secret will be used to restore the unseal + # shards for the openbao that is being restored. + function snapshotRestore { + local metadata="$1" + local secret + local logs + local result + local initdata + local apipath="/sys/storage/raft/snapshot-force" + + # check that the associated secret exists + secret="$( echo "$metadata" | jq -r .secret 2>/dev/null )" + if [ -z "$secret" -o "$secret" == "null" ]; then + log $ERROR "Metadata omits the k8s secret associated with" \ + "the snapshot" + return 1 + fi + + secretExists "$secret" >/dev/null + if [ $? -ne 0 ]; then + log $ERROR "K8s secret [$secret] associated with the" \ + "snapshot does not exist" + return 1 + fi + + # check the init response associated with the snapshot + initdata="$( get_secret "$secret" | jq -c .init 2>/dev/null )" + if [ -z "$initdata" -o "$initdata" == 'null' ]; then + log $ERROR "Failed to retrieve init response from" \ + "k8s secret [$secret]" + return 1 + fi + + # The snapshot API success does not give a response. On openbao + # API error the return code is also 0. If there is a log, then + # there was an error. + logs="$( curl -s -S --cacert "$CERT" \ + --connect-timeout $QUERY_TMOUT \ + --header "X-Vault-Token:$( get_secret cluster-key-root )" \ + --request POST \ + --data-binary @/dev/stdin \ + "https://$ACTIVE_TARGET:${TARGET_PORT}/v1${apipath}" 2>&1 )" + + result=$? + log $INFO "Snapshot restore API response: $result" + if [ "$result" -ne 0 -o -n "$logs" ]; then + log $ERROR "Snapshot restore: [$logs]" + return 1 + fi + + # Restore the secrets associated with the snapshot + # We're done if the secrets haven't changed. + echo "$initdata" | validateSecrets cluster-key + if [ $? -eq 0 ]; then + return 0 + fi + + # replace openbao's init response in k8s secrets + deleteShardSecrets cluster-key + deleteSecrets cluster-key-root + echo "$initdata" | storeOpenbaoInitSecrets cluster-key + + # finally, verify the storage was successful + echo "$initdata" | validateSecrets cluster-key + return $? + } + + # function that calls exit_on_trap for every second of sleep + # takes total sleep time as parameter + function trap_sleep { + local sleep_time="$1" + + for i in $(seq 1 $sleep_time); do + sleep 1 + exit_on_trap 22 + done + } + + + # + # LOGIC + # + if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + # This script was sourced + return 0 + fi + + health_excuse_create "$HEALTH_EXCUSE_INIT" "$HC_MSG_INIT" + if [ -n "$EARLY_PAUSE" ]; then + echo -n "$EARLY_PAUSE" > $PAUSEFILE + fi + + exit_on_trap 1 + + # Match kubectl version to server version (or etc) + pickK8sVersion + + # check if this pod is helping to convert storage from pvc to k8s + # secrets + mountHelper + exit_on_trap 15 + + # check if there are existing key shard secrets, boot strap secret, + # or pre-existing resource + K8S_SECRETS_PREEXIST="$( secretExists cluster-key-root )" + exit_on_trap 16 + BOOTSTRAP_PREEXISTS="$( secretExists cluster-key-bootstrap )" + exit_on_trap 17 + PVC_PREEXISTS="$( pvcRemoved )" + exit_on_trap 18 + + runConversion + exit_on_trap 19 + + # check if PVC still persisted after conversion, and if so issue a warning. + PVC_PREEXISTS="$( pvcRemoved )" + PVC_STATUS=$? + if [ $PVC_STATUS -eq 1 ]; then + log $DEBUG "PVC storage $PVC_PREEXISTS is currently terminating" + elif [ $PVC_STATUS -eq 2 ]; then + log $WARNING "PVC storage $PVC_PREEXISTS deletion has failed during conversion" + fi + + # Waiting for at least one openbao server, to check initialization + waitForPods 1 + exit_on_trap 2 + + log $DEBUG "Putting a list of openbao pods and ip in $WORKDIR/pods.txt" + getOpenbaoPods > $WORKDIR/pods.txt + exit_on_trap 3 + + openbaoInitialized + IS_OPENBAO_INITIALIZED=$? + if [ $IS_OPENBAO_INITIALIZED -eq 1 ]; then + exit_on_trap 4 + desired_pods=$HA_REPLICAS + + # Waiting for openbao servers to come up + waitForPods $desired_pods + exit_on_trap 5 + + log $INFO "Putting a list of openbao pods and IPs in $WORKDIR/pods.txt" + getOpenbaoPods > $WORKDIR/pods.txt + exit_on_trap 6 + + log $DEBUG "Initializing the openbao on openbao-0 and" \ + "storing keys in k8s secrets" + initOpenbao + + #Some sleep required to allow convergence" + sleep "$INIT_CONVERGE_TIME" + + log $DEBUG "Unsealing openbao-0 using the init shards" + for row in $(awk 'NR==1{print $2}' $WORKDIR/pods.txt); do + unsealOpenbao "$row" + done + + log $DEBUG "Joining other openbao servers to the HA Raft cluster" + for row in $(awk 'NR>1{print $2}' $WORKDIR/pods.txt); do + log $DEBUG "$( grep $row $WORKDIR/pods.txt )" + joinRaft "$row" + sleep "$JOIN_RATE" + done + + exit_on_trap 7 + log $INFO "Unsealing the remaining openbaos" + for row in $(awk 'NR>1{print $2}' $WORKDIR/pods.txt); do + log $DEBUG "$( grep $row $WORKDIR/pods.txt )" + unsealOpenbao "$row" + sleep "$UNSEAL_RATE" + exit_on_trap 8 + done + else + log $INFO "Openbao is initialized" + fi + + exit_on_trap 9 + # initialize the state machine - openbao server status records + echo "" > "$PODREC_F" + while read host dns_name; do + if [ -z "$host" ]; then + continue + fi + status_rec="/$host/$dns_name//" + echo "$status_rec" >> "$PODREC_F" + done <$WORKDIR/pods.txt + + health_excuse_remove "$HEALTH_EXCUSE_INIT" + + # Loop forever to check the seal status of openbaos and + # unseal if required + log $INFO "Checking openbao pods seal status in perpetuity..." + while true; do + exit_on_trap 10 + trap_sleep "$STATUS_RATE" + exit_on_trap 20 + pickK8sVersion # check if the k8s server version is changed + + count=$( kubectl get pods -n "${OPENBAO_NS}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + | grep "^${OPENBAO_FN}-manager" | wc -w ) + if [ "$count" -gt 1 ]; then + log $ERROR "Multiple instances of openbao manager detected. Waiting until one left" + exit_on_trap 21 + continue + fi + + rm $WORKDIR/pods.txt + echo "" > "$PODREC_TMP_F" + exit_on_trap 11 + getOpenbaoPods > $WORKDIR/pods.txt + exit_on_trap 12 + + while read host dnsname; do + if [ -z "$dnsname" ]; then + # probably a recovering pod waiting for an IP address + log $DEBUG "pod list has empty data: [$host] [$dnsname]" + continue + fi + + NO_HEADER=true \ + API_TMOUT=$QUERY_TMOUT \ + openbaoAPI server_status GET $dnsname.$POD_TARGET_BASE \ + /sys/health + echo -n "$server_status" > $WORKDIR/healthcheck.txt + + TEMP=$( echo "$server_status" | jq -r .sealed ) + + exit_on_trap 13 + # Decide when to unseal the openbao server; includes + # Adding records to new_pods_status.txt + runStateMachine "$host" "$dnsname" "$TEMP" + exit_on_trap 14 + done <$WORKDIR/pods.txt + mv "$PODREC_TMP_F" "$PODREC_F" + + openbaoRekey + done +kind: ConfigMap +metadata: + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:init.sh: {} + manager: openbao-init-unseal + name: openbao-init-unseal-3 + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:pvc-attach.yaml: {} + manager: {{ .Values.openbao.name }}-mount-helper + name: {{ .Values.openbao.name }}-mount-helper + namespace: {{ .Release.Namespace }} +data: + pvc-attach.yaml: | + --- + apiVersion: batch/v1 + kind: Job + metadata: + name: {{ .Values.openbao.fullname }}-mount-helper + namespace: openbao + spec: + activeDeadlineSeconds: 600 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 0 + template: + spec: + restartPolicy: Never + serviceAccountName: "{{ .Values.openbao.fullname }}-manager-1" + {{- if .Values.manager.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.manager.imagePullSecrets | nindent 12 }} + {{- end }} + {{- if .Values.manager.tolerations }} + tolerations: + {{- tpl .Values.manager.tolerations . | nindent 12 }} + {{- end }} + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: mount + image: "{{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }}" + imagePullPolicy: "{{ .Values.manager.image.pullPolicy }}" + args: + - bash + - /opt/script/init.sh + env: + - name: MANAGER_MODE + value: MOUNT_HELPER + - name: PVC_DIR + value: /mnt/data + volumeMounts: + - name: mount-helper + mountPath: /opt/script + readOnly: true + - name: manager-pvc + mountPath: /mnt/data + readOnly: false + volumes: + - name: mount-helper + configMap: + name: openbao-init-unseal-3 + - name: manager-pvc + persistentVolumeClaim: + claimName: manager-pvc-stx-openbao-manager-0 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: {{ .Release.Namespace }} + name: {{ .Values.openbao.fullname }}-manager-1 +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] # "" indicates the core API group + resources: ["pods/exec"] + verbs: ["create"] +- apiGroups: [""] # "" indicates the core API group + resources: ["secrets"] + verbs: ["get", "create", "delete"] +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "create", "delete"] +- apiGroups: [""] # "" indicates the core API group + resources: ["persistentvolumeclaims"] + verbs: ["list", "delete"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.openbao.fullname }}-manager-1 + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ .Values.manager.chart }} + app.kubernetes.io/name: {{ .Values.openbao.name }}-manager + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Values.openbao.fullname }}-manager-1 + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ .Values.openbao.fullname }}-manager-1 +roleRef: + kind: Role + name: {{ .Values.openbao.fullname }}-manager-1 + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Values.openbao.fullname }}-manager-3 + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ .Values.openbao.name }}-manager + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + component: webhook +spec: + serviceName: {{ .Values.openbao.fullname }} + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + component: webhook + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Values.openbao.name }}-manager + app.kubernetes.io/instance: {{ .Release.Name }} + app.starlingx.io/component: {{ ternary "application" "platform" .Values.manager.labels.isApplication}} + component: webhook + annotations: { + configchecksum: {{ toYaml .Values.manager.labels | sha256sum | trunc 63 }} + } + spec: + serviceAccountName: "{{ .Values.openbao.fullname }}-manager-1" + {{- if .Values.manager.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.manager.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.manager.tolerations }} + tolerations: + {{- tpl .Values.manager.tolerations . | nindent 8 }} + {{- end }} + containers: + - name: manager + image: "{{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }}" + imagePullPolicy: "{{ .Values.manager.image.pullPolicy }}" + args: + - bash + - /opt/script/init.sh + env: + - name: CA_CERT + value: /mnt/data/ca/tls.crt + livenessProbe: + exec: + command: + - bash + - -c + - "source /opt/script/init.sh; health_check" + initialDelaySeconds: {{ .Values.manager.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.manager.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.manager.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.manager.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.manager.livenessProbe.failureThreshold }} + terminationGracePeriodSeconds: {{ .Values.manager.livenessProbe.terminationGracePeriodSeconds }} + volumeMounts: + - name: openbao-init-unseal-3 + mountPath: /opt/script + readOnly: false + - name: mount-helper-yaml + mountPath: /opt/yaml + readOnly: true + - name: openbao-ca + mountPath: /mnt/data/ca + readOnly: true + volumes: + - name: openbao-init-unseal-3 + configMap: + name: openbao-init-unseal-3 + - name: mount-helper-yaml + configMap: + name: {{ .Values.openbao.name }}-mount-helper + - name: openbao-ca + secret: + secretName: openbao-ca diff --git a/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/values.yaml b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/values.yaml new file mode 100644 index 0000000..e212fb3 --- /dev/null +++ b/helm-charts/custom/openbao-manager-helm/openbao-manager-helm/openbao-manager/values.yaml @@ -0,0 +1,184 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Values migrated from openbao helm chart + +openbao: + name: openbao + fullname: stx-openbao + +server: + version: 2.1.0 + ha: + replicas: 1 + +# Openbao Manager specific values +manager: + image: + repository: starlingx/stx-vault-manager + tag: stx.10.0-v1.29.6-1 + pullPolicy: IfNotPresent + + chart: openbao_1.0.1 + + imagePullSecrets: [] + + # Rate at which openbao-manager checks status of openbao servers. + # After initialization of the raft, Openbao manager will loop forever + # checking the pods for openbao servers that need to be unsealed. + # This value is the sleep, in seconds, between intervals. Value + # must be a positive integer + statusCheckRate: 5 + + # After initial configuration, in combination with statusCheckRate, + # the amount of time to wait before unsealing a recovering openbao + # server. The option is intended to allow the active openbao server + # time to start sending heartbeats to the recovering pod before + # unsealing the server. + # + # A value of 0 indicates no wait time: unseal the openbao server without + # delay. The wait time is statusCheckRate * unsealWaitIntervals. + # Default is 5 s/interval * 3 intervals == 15 seconds. + # + unsealWaitIntervals: 3 + + api: + # Network timeout for queries to openbao server /sys/health endpoint + # + # The maximum time in seconds to wait for a server to respond to + # health query. This applies for the HA recovery situations, not the + # initialization of openbao cluster. Unsetting the value is not + # recommended, and defaults to timeout of 120 seconds. + # + # openbao-manager will appear to hang if healthQueryTimeout is + # over-large. This setting affects the logs, since openbao-manager will + # issue a log when the 'sealed' status toggles between true/false and + # the 'unknown' value + healthQueryTimeout: 2 + + # Network timeout for openbao API operations against /sys/unseal + # + # The maximum time in seconds to wait for a server to respond to + # the unseal request. + unsealOpTimeout: 10 + + # Network timeout for queries to openbao server /sys/rekey/init + # and /sys/rekey/verify + # + # The maximum time in seconds to wait for a server to respond to + # the query. + rekeyStatusTimeout: 2 + + # Network timeout for openbao API operations against /sys/rekey/init + # and /sys/rekey/verify + # + # The maximum time in seconds to wait for a server to respond to + # the request. + rekeyOpTimeout: 10 + + rekey: + # During upgrade of the application from PVC storage to storage + # using kubernetes, enable openbao rekey to run automatically to + # resecure the openbao with new shards. + # See also openbao documentation: + # https://openbao.org/docs/concepts/seal/#rekeying + # https://openbao.org/api-docs/system/rekey + # + enableOnPVCConversion: true + + k8s: + # The major/minor version of kubectl client binary to use. Must + # exist within the openbao manager image for example + # client_version: v1.28 + client_version: "" + + waitTermination: + # During upgrade of the application from PVC storage to storage + # using kubernetes, wait for previous version of openbao manager + # to terminate before proceding with the conversion of storage from PVC to + # kubernetes secrets. + # + # The maximum tries before proceding with the conversion of storage + # from PVC to kubernetes secrets. + maxTries: 12 + + # Number of seconds slept between each tries before proceding with + # the conversion of storage from PVC to kubernetes secrets. + sleepTime: 5 + + # Labeling pods for StarlingX core management. Setting 'true' will schedule pods to be run on + # application cores, while setting 'false' will schedule pods to be run on platform cores. + labels: + isApplication: false + + # Request openbao-manager to pause on startup. + # + # The pause feature allows execution of openbao-manager to be suspended + # for external operations or for debugging. A pause_on_trap file will + # be created with the content of this value. Values may include a + # positive integer matching a call of exit_on_trap + # + # pause: 1 + + # Debugging option to improve log reading, allow more verbose logging + # DEBUG: 1 + # INFO: 2 + # WARNING: 3 + # ERROR: 4 + # FATAL: 5 + log: + defaultLogLevel: 2 + + # All options related to openbao manager healthcheck funtion + healthcheck: + # Disables the healthcheck function. It will always return as "healthy" + # When this is set to true + disableHC: false + + # Enables the network excuse of the healthcheck function. When enabled, + # healthcheck function will always return as "healthy" when openbao manager + # is accessing openbao REST API + enableNetwork: true + + # Enables the initialization excuse of the healthcheck funtion. When + # enabled, healthcheck function will always return as "healthy" when openbao + # manager is initializing + enableInit: true + + # Enables the pause excuse of the healthcheck function. when enabled, + # healthcheck function will always return as "healthy" when openbao manager + # is paused during exit_on_trap by the pause option. + enablePause: true + + # Maximum threshold in seconds, between the last heartbeat and healthcheck. + # If the time passed between the last heartbeat and the latest healthcheck + # passes the threshold value, then the healthcheck will fail if no excuses + # were found. + heartbeatThreshold: 30 + + # All options related to openbao manager liveness probe. Consult the kubernetes + # documentation to find more details on each options. + livenessProbe: + + # Number of seconds before the first probe is initiated + initialDelaySeconds: 0 + + # Number of seconds between each probe + periodSeconds: 10 + + # Number of seconds before the probe times out + timeoutSeconds: 1 + + # Number of successful probes required for the pod to be considered + # successful. + successThreshold: 1 + + # Number of failed probes required for the pod to be considered failed + failureThreshold: 3 + + # Number of seconds to wait from triggering the shutdown on the container, + # and the forced stop of the container by the container runtime. + terminationGracePeriodSeconds: 30 diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/changelog b/helm-charts/upstream/openbao-helm/debian/deb_folder/changelog new file mode 100644 index 0000000..92badad --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/changelog @@ -0,0 +1,5 @@ +openbao-helm (0.7-0) unstable; urgency=medium + + * Initial release. + + -- Tae Park Thu, 10 Oct 2024 19:34:18 -0400 diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/control b/helm-charts/upstream/openbao-helm/debian/deb_folder/control new file mode 100644 index 0000000..313c4c0 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/control @@ -0,0 +1,16 @@ +Source: openbao-helm +Section: libs +Priority: optional +Maintainer: StarlingX Developers +Build-Depends: debhelper-compat (= 13), + helm, + build-info, +Standards-Version: 4.5.1 +Homepage: https://www.starlingx.io + +Package: openbao-helm +Section: libs +Architecture: any +Depends: ${misc:Depends} +Description: StarlingX Openbao Helm Charts + This package contains helm charts for the openbao application. diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/copyright b/helm-charts/upstream/openbao-helm/debian/deb_folder/copyright new file mode 100644 index 0000000..3d366b3 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/copyright @@ -0,0 +1,48 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: Openbao-helm +Source: https://opendev.org/starlingx/app-openbao/ + +Files: * +Copyright: (c) 2024 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. + +# If you want to use GPL v2 or later for the /debian/* files use +# the following clauses, or change it to suit. Delete these two lines +Files: debian/* +Copyright: 2024 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. + +Files: usr/lib/helm/* +Copyright: 2024 Openbao +License: MPL 2.0 + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. \ No newline at end of file diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/openbao-helm.install b/helm-charts/upstream/openbao-helm/debian/deb_folder/openbao-helm.install new file mode 100644 index 0000000..8a0c6de --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/openbao-helm.install @@ -0,0 +1 @@ +usr/lib/helm/* diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0001-Add-yaml-for-Starlingx-image-handling.patch b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0001-Add-yaml-for-Starlingx-image-handling.patch new file mode 100644 index 0000000..b49078c --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0001-Add-yaml-for-Starlingx-image-handling.patch @@ -0,0 +1,33 @@ +From e1040978fd030a5d2b8f623c59adf3a3764c9c33 Mon Sep 17 00:00:00 2001 +From: Tae Park +Date: Mon, 21 Oct 2024 09:30:07 -0400 +Subject: [PATCH] Add yaml for Starlingx image handling + +Add values yaml compatible with Starlingx platform's image pull and +service parameter registry override handling. The platform will pull +the image and populate registry.local, and the injector agent will +pull from registry.local. + +Signed-off-by: Tae Park +--- + charts/openbao/values.yaml | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/charts/openbao/values.yaml b/charts/openbao/values.yaml +index d9c59a6..bc82fc8 100644 +--- a/charts/openbao/values.yaml ++++ b/charts/openbao/values.yaml +@@ -79,6 +79,10 @@ injector: + # containers. This should be set to the official OpenBao image. OpenBao 1.3.1+ is + # required. + agentImage: ++ image: ++ registry: "quay.io" ++ repository: "openbao/openbao" ++ tag: "2.1.0" + # -- image registry to use for agent image + registry: "quay.io" + # -- image repo to use for agent image +-- +2.34.1 + diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0002-update-includeConfigAnnotation-to-match-README-value.patch b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0002-update-includeConfigAnnotation-to-match-README-value.patch new file mode 100644 index 0000000..8b91d97 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0002-update-includeConfigAnnotation-to-match-README-value.patch @@ -0,0 +1,74 @@ +From 51bbb43dd02b0be89b05549ebae6107fb5ab4e29 Mon Sep 17 00:00:00 2001 +From: Michel Thebeau +Date: Mon, 13 Jan 2025 16:01:08 -0500 +Subject: [PATCH] update includeConfigAnnotation to match README/values + +The value of server.includeConfigAnnotation is presented in the +README.md and values.yaml as server.configAnnotation. + +Update the server configmap, _helpers.tpl and unit tests to align with +the documentation. + +Signed-off-by: Michel Thebeau +--- + charts/openbao/templates/_helpers.tpl | 2 +- + charts/openbao/templates/server-config-configmap.yaml | 2 +- + test/unit/server-configmap.bats | 2 +- + test/unit/server-statefulset.bats | 2 +- + 4 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/charts/openbao/templates/_helpers.tpl b/charts/openbao/templates/_helpers.tpl +index 2650db5..08c71ee 100644 +--- a/charts/openbao/templates/_helpers.tpl ++++ b/charts/openbao/templates/_helpers.tpl +@@ -448,7 +448,7 @@ Sets extra pod annotations + */}} + {{- define "openbao.annotations" }} + annotations: +- {{- if .Values.server.includeConfigAnnotation }} ++ {{- if .Values.server.configAnnotation }} + openbao.hashicorp.com/config-checksum: {{ include "openbao.config" . | sha256sum }} + {{- end }} + {{- if .Values.server.annotations }} +diff --git a/charts/openbao/templates/server-config-configmap.yaml b/charts/openbao/templates/server-config-configmap.yaml +index 585ae7a..57c4b0e 100644 +--- a/charts/openbao/templates/server-config-configmap.yaml ++++ b/charts/openbao/templates/server-config-configmap.yaml +@@ -18,7 +18,7 @@ metadata: + app.kubernetes.io/name: {{ include "openbao.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +-{{- if .Values.server.includeConfigAnnotation }} ++{{- if .Values.server.configAnnotation }} + annotations: + vault.hashicorp.com/config-checksum: {{ include "openbao.config" . | sha256sum }} + {{- end }} +diff --git a/test/unit/server-configmap.bats b/test/unit/server-configmap.bats +index 55d67e9..aa34edc 100755 +--- a/test/unit/server-configmap.bats ++++ b/test/unit/server-configmap.bats +@@ -153,7 +153,7 @@ load _helpers + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-config-configmap.yaml \ +- --set 'server.includeConfigAnnotation=true' \ ++ --set 'server.configAnnotation=true' \ + . | tee /dev/stderr | + yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr) + [ "${actual}" = "false" ] +diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats +index 9a4bf3e..f089252 100755 +--- a/test/unit/server-statefulset.bats ++++ b/test/unit/server-statefulset.bats +@@ -1636,7 +1636,7 @@ load _helpers + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ +- --set 'server.includeConfigAnnotation=true' \ ++ --set 'server.configAnnotation=true' \ + . | tee /dev/stderr | + yq '.spec.template.metadata.annotations["openbao.hashicorp.com/config-checksum"] == null' | tee /dev/stderr) + [ "${actual}" = "false" ] +-- +2.34.1 + diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0003-Fix-helm-template-for-server-annotations.patch b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0003-Fix-helm-template-for-server-annotations.patch new file mode 100644 index 0000000..d0fa007 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0003-Fix-helm-template-for-server-annotations.patch @@ -0,0 +1,33 @@ +From d92aafc376d718d1b0cd802b51129839c9f5b01a Mon Sep 17 00:00:00 2001 +From: Tae Park +Date: Mon, 16 Dec 2024 14:51:11 -0500 +Subject: [PATCH] Fix helm template for server annotations + +The current version of openbao-helm has bug in the server annotation +template. A values.yaml with empty server annotation produces a +server-statefulset.yaml containing a key +spec.template.metadata.annotations with no assigned value. This patch +updates the logic in the template to not create empty yaml key. + +Signed-off-by: Tae Park +--- + charts/openbao/templates/_helpers.tpl | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/charts/openbao/templates/_helpers.tpl b/charts/openbao/templates/_helpers.tpl +index 08c71ee..1fecef9 100644 +--- a/charts/openbao/templates/_helpers.tpl ++++ b/charts/openbao/templates/_helpers.tpl +@@ -447,7 +447,9 @@ Sets the injector deployment update strategy + Sets extra pod annotations + */}} + {{- define "openbao.annotations" }} ++ {{- if or (.Values.server.annotations) (.Values.server.configAnnotation) }} + annotations: ++ {{- end }} + {{- if .Values.server.configAnnotation }} + openbao.hashicorp.com/config-checksum: {{ include "openbao.config" . | sha256sum }} + {{- end }} +-- +2.34.1 + diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0004-Fix-agent-image-registry-in-injector-deployment.patch b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0004-Fix-agent-image-registry-in-injector-deployment.patch new file mode 100644 index 0000000..5364e11 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/0004-Fix-agent-image-registry-in-injector-deployment.patch @@ -0,0 +1,31 @@ +From b98367b9f629c42fae00e4288127d7559943c2a4 Mon Sep 17 00:00:00 2001 +From: Tae Park +Date: Fri, 10 Jan 2025 09:50:23 -0500 +Subject: [PATCH] Fix agent image registry in injector deployment + +The value for AGENT_INJECT_VAULT_IMAGE in injector-deployment.yaml +incorrectly points to injector image's registry instead of the agent +image registry. This changes the registry to the correct agent image +one, so the agent image variable points to the correct image. + +Signed-off-by: Tae Park +--- + charts/openbao/templates/injector-deployment.yaml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/charts/openbao/templates/injector-deployment.yaml b/charts/openbao/templates/injector-deployment.yaml +index 64e0de2..d66f6d1 100644 +--- a/charts/openbao/templates/injector-deployment.yaml ++++ b/charts/openbao/templates/injector-deployment.yaml +@@ -69,7 +69,7 @@ spec: + - name: AGENT_INJECT_VAULT_AUTH_PATH + value: {{ .Values.injector.authPath }} + - name: AGENT_INJECT_VAULT_IMAGE +- value: "{{ .Values.injector.image.registry | default "quay.io" }}/{{ .Values.injector.agentImage.repository }}:{{ .Values.injector.agentImage.tag }}" ++ value: "{{ .Values.injector.agentImage.registry | default "quay.io" }}/{{ .Values.injector.agentImage.repository }}:{{ .Values.injector.agentImage.tag }}" + {{- if .Values.injector.certs.secretName }} + - name: AGENT_INJECT_TLS_CERT_FILE + value: "/etc/webhook/certs/{{ .Values.injector.certs.certName }}" +-- +2.34.1 + diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/series b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/series new file mode 100644 index 0000000..187430f --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/patches/series @@ -0,0 +1,4 @@ +0001-Add-yaml-for-Starlingx-image-handling.patch +0002-update-includeConfigAnnotation-to-match-README-value.patch +0003-Fix-helm-template-for-server-annotations.patch +0004-Fix-agent-image-registry-in-injector-deployment.patch diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/rules b/helm-charts/upstream/openbao-helm/debian/deb_folder/rules new file mode 100755 index 0000000..2562dc4 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/rules @@ -0,0 +1,29 @@ +#!/usr/bin/make -f +export DH_VERBOSE = 1 + +export DEB_VERSION = $(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') +export PATCH_VERSION = $(shell echo $(DEB_VERSION) | cut -f 4 -d '.') +export CHART_BASE_VERSION = $(shell echo $(DEB_VERSION) | sed 's/-/./' | cut -d '.' -f 1-3) +export CHART_VERSION = $(CHART_BASE_VERSION)+STX.$(PATCH_VERSION) + +export ROOT = debian/tmp +export APP_FOLDER = $(ROOT)/usr/lib/helm + +%: + dh $@ + +override_dh_auto_build: + # Set up chart build files. + mkdir openbao + cp charts/openbao/Chart.yaml charts/openbao/values.yaml openbao + cp openbao-certificates.yaml charts/openbao/templates + mv charts/openbao/templates openbao/templates + # Create the TGZ file. + make CHART_VERSION=$(CHART_VERSION) openbao + +override_dh_auto_install: + # Install the app tar file. + install -d -m 755 $(APP_FOLDER) + install -p -D -m 755 openbao*.tgz $(APP_FOLDER) + +override_dh_auto_test: diff --git a/helm-charts/upstream/openbao-helm/debian/deb_folder/source/format b/helm-charts/upstream/openbao-helm/debian/deb_folder/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/deb_folder/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/helm-charts/upstream/openbao-helm/debian/meta_data.yaml b/helm-charts/upstream/openbao-helm/debian/meta_data.yaml new file mode 100644 index 0000000..cd783a0 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/debian/meta_data.yaml @@ -0,0 +1,14 @@ +--- +debname: openbao-helm +debver: 0.7-0 +dl_path: + name: openbao-0.7.0.tar.gz + url: https://github.com/openbao/openbao-helm/archive/refs/tags/openbao-0.7.0.tar.gz + sha256sum: ade5a12443a76a98a65764853ab6d1840d45654a9b3e1aa6098c1b59b4847dfc +src_files: + - openbao-helm/files/Makefile + - openbao-helm/helm-charts/openbao-certificates.yaml +revision: + dist: $STX_DIST + PKG_GITREVCOUNT: + PKG_BASE_SRCREV: 0144b018a6a592860dace539e9fb937af7b2d26f diff --git a/helm-charts/upstream/openbao-helm/openbao-helm/README b/helm-charts/upstream/openbao-helm/openbao-helm/README new file mode 100644 index 0000000..b773ef4 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/openbao-helm/README @@ -0,0 +1,5 @@ +This directory contains all StarlingX charts that need to be built for this +application. Some charts are common across applications. These common charts +reside in the stx-config/kubernetes/helm-charts directory. To include these in +this application update the build_srpm.data file and use the COPY_LIST_TO_TAR +mechanism to populate these common charts. diff --git a/helm-charts/upstream/openbao-helm/openbao-helm/files/Makefile b/helm-charts/upstream/openbao-helm/openbao-helm/files/Makefile new file mode 100644 index 0000000..b65d9fe --- /dev/null +++ b/helm-charts/upstream/openbao-helm/openbao-helm/files/Makefile @@ -0,0 +1,41 @@ +# +# Copyright 2017 The Openstack-Helm Authors. +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# It's necessary to set this because some environments don't link sh -> bash. +SHELL := /bin/bash +TASK := build + +EXCLUDES := doc tests tools logs tmp +CHARTS := $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.))) + +.PHONY: $(EXCLUDES) $(CHARTS) + +all: $(CHARTS) + +$(CHARTS): + @if [ -d $@ ]; then \ + echo; \ + echo "===== Processing [$@] chart ====="; \ + make $(TASK)-$@; \ + fi + +init-%: + if [ -f $*/Makefile ]; then make -C $*; fi + +lint-%: init-% + if [ -d $* ]; then helm lint $*; fi + +build-%: + if [ -d $* ]; then helm package --version $(CHART_VERSION) $*; fi + +clean: + @echo "Clean all build artifacts" + rm -f */templates/_partials.tpl */templates/_globals.tpl + rm -rf */charts */tmpcharts + +%: + @: diff --git a/helm-charts/upstream/openbao-helm/openbao-helm/helm-charts/openbao-certificates.yaml b/helm-charts/upstream/openbao-helm/openbao-helm/helm-charts/openbao-certificates.yaml new file mode 100644 index 0000000..ac84f19 --- /dev/null +++ b/helm-charts/upstream/openbao-helm/openbao-helm/helm-charts/openbao-certificates.yaml @@ -0,0 +1,73 @@ +{{ $ca := genCA "svc-cat-ca" 3650 }} + +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: {{ template "openbao.name" . }}-ca + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "openbao.name" . }} + chart: {{ template "openbao.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" +data: + tls.crt: {{ b64enc $ca.Cert }} + tls.key: {{ b64enc $ca.Key }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + creationTimestamp: null + name: ca-issuer + namespace: {{ .Release.Namespace }} +spec: + ca: + secretName: {{ template "openbao.name" . }}-ca +status: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + creationTimestamp: null + name: openbao-server-tls + namespace: {{ .Release.Namespace }} +spec: + # Secret names are always required. + secretName: openbao-server-tls + duration: 2160h0m0s # 90d + renewBefore: 360h0m0s # 15d + usages: + - server auth + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - stx-{{ template "openbao.name" . }} + - '*.stx-{{ template "openbao.name" . }}-internal' + - '*.{{ .Release.Namespace }}.pod.cluster.local' + - stx-{{ template "openbao.name" . }}.{{ .Release.Namespace }} + - stx-{{ template "openbao.name" . }}.{{ .Release.Namespace }}.svc + - stx-{{ template "openbao.name" . }}.{{ .Release.Namespace }}.svc.cluster.local + - stx-{{ template "openbao.name" .}}-active.{{ .Release.Namespace }}.svc.cluster.local + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: ca-issuer + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: Issuer + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + subject: + organizations: + - stx +status: {} diff --git a/python3-k8sapp-openbao/debian/deb_folder/changelog b/python3-k8sapp-openbao/debian/deb_folder/changelog new file mode 100644 index 0000000..7b98d04 --- /dev/null +++ b/python3-k8sapp-openbao/debian/deb_folder/changelog @@ -0,0 +1,5 @@ +python3-k8sapp-openbao (1.0-1) unstable; urgency=medium + + * Initial release. + + -- Tae Park Thu, 10 Oct 2024 19:34:18 -0400 diff --git a/python3-k8sapp-openbao/debian/deb_folder/control b/python3-k8sapp-openbao/debian/deb_folder/control new file mode 100644 index 0000000..7eb4edc --- /dev/null +++ b/python3-k8sapp-openbao/debian/deb_folder/control @@ -0,0 +1,27 @@ +Source: python3-k8sapp-openbao +Section: libs +Priority: optional +Maintainer: StarlingX Developers +Build-Depends: debhelper-compat (= 13), + dh-python, + python3-all, + python3-pbr, + python3-setuptools, + python3-wheel, + build-info +Standards-Version: 4.5.1 +Homepage: https://www.starlingx.io + +Package: python3-k8sapp-openbao +Section: libs +Architecture: any +Depends: ${misc:Depends}, ${python3:Depends} +Description: StarlingX Sysinv Openbao Extensions + This package contains sysinv plugins for the openbao K8S app. + +Package: python3-k8sapp-openbao-wheels +Section: libs +Architecture: any +Depends: ${misc:Depends}, ${python3:Depends}, python3-wheel +Description: StarlingX Sysinv Openbao Extension Wheels + This package contains python wheels for the openbao K8S app plugins. diff --git a/python3-k8sapp-openbao/debian/deb_folder/copyright b/python3-k8sapp-openbao/debian/deb_folder/copyright new file mode 100644 index 0000000..5febe53 --- /dev/null +++ b/python3-k8sapp-openbao/debian/deb_folder/copyright @@ -0,0 +1,41 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: python3-k8sapp-openbao +Source: https://opendev.org/starlingx/app-openbao/ + +Files: * +Copyright: (c) 2024 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. + +# If you want to use GPL v2 or later for the /debian/* files use +# the following clauses, or change it to suit. Delete these two lines +Files: debian/* +Copyright: 2024 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. diff --git a/python3-k8sapp-openbao/debian/deb_folder/python3-k8sapp-openbao-wheels.install b/python3-k8sapp-openbao/debian/deb_folder/python3-k8sapp-openbao-wheels.install new file mode 100644 index 0000000..19a9e4c --- /dev/null +++ b/python3-k8sapp-openbao/debian/deb_folder/python3-k8sapp-openbao-wheels.install @@ -0,0 +1 @@ +plugins/*.whl diff --git a/python3-k8sapp-openbao/debian/deb_folder/python3-k8sapp-openbao.install b/python3-k8sapp-openbao/debian/deb_folder/python3-k8sapp-openbao.install new file mode 100644 index 0000000..91d1d9d --- /dev/null +++ b/python3-k8sapp-openbao/debian/deb_folder/python3-k8sapp-openbao.install @@ -0,0 +1 @@ +usr/lib/python3/dist-packages/k8sapp_* diff --git a/python3-k8sapp-openbao/debian/deb_folder/rules b/python3-k8sapp-openbao/debian/deb_folder/rules new file mode 100755 index 0000000..9b5c2d1 --- /dev/null +++ b/python3-k8sapp-openbao/debian/deb_folder/rules @@ -0,0 +1,33 @@ +#!/usr/bin/make -f +# export DH_VERBOSE = 1 + +export APP_NAME = openbao +export PYBUILD_NAME = k8sapp-openbao + +export DEB_VERSION = $(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') +export MAJOR = $(shell cat /etc/build.info | grep SW_VERSION | cut -d'"' -f2) +export MINOR_PATCH = $(shell echo $(DEB_VERSION) | cut -f 4 -d '.') +export PBR_VERSION = $(MAJOR).$(MINOR_PATCH) + +export ROOT = $(CURDIR)/debian/tmp +export SKIP_PIP_INSTALL = 1 + +%: + dh $@ --with=python3 --buildsystem=pybuild + +override_dh_auto_install: + env | sort + + python3 setup.py install \ + --install-layout=deb \ + --root $(ROOT) + + python3 setup.py bdist_wheel \ + --universal \ + -d $(ROOT)/plugins + +override_dh_python3: + dh_python3 --shebang=/usr/bin/python3 + +override_dh_auto_test: + PYTHONDIR=$(CURDIR) stestr run diff --git a/python3-k8sapp-openbao/debian/deb_folder/source/format b/python3-k8sapp-openbao/debian/deb_folder/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/python3-k8sapp-openbao/debian/deb_folder/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/python3-k8sapp-openbao/debian/meta_data.yaml b/python3-k8sapp-openbao/debian/meta_data.yaml new file mode 100644 index 0000000..347def8 --- /dev/null +++ b/python3-k8sapp-openbao/debian/meta_data.yaml @@ -0,0 +1,9 @@ +--- +debname: python3-k8sapp-openbao +debver: 1.0-1 +src_path: k8sapp_openbao +revision: + dist: $STX_DIST + SRC_GITREVCOUNT: + SRC_DIR: ${MY_REPO}/stx/app-openbao + SRC_BASE_SRCREV: 0144b018a6a592860dace539e9fb937af7b2d26f diff --git a/python3-k8sapp-openbao/k8sapp_openbao/.gitignore b/python3-k8sapp-openbao/k8sapp_openbao/.gitignore new file mode 100644 index 0000000..78c457c --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/.gitignore @@ -0,0 +1,35 @@ +# Compiled files +*.py[co] +*.a +*.o +*.so + +# Sphinx +_build +doc/source/api/ + +# Packages/installer info +*.egg +*.egg-info +dist +build +eggs +parts +var +sdist +develop-eggs +.installed.cfg + +# Other +*.DS_Store +.stestr +.testrepository +.tox +.venv +.*.swp +.coverage +bandit.xml +cover +AUTHORS +ChangeLog +*.sqlite diff --git a/python3-k8sapp-openbao/k8sapp_openbao/.stestr.conf b/python3-k8sapp-openbao/k8sapp_openbao/.stestr.conf new file mode 100644 index 0000000..df136cb --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/.stestr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_path=./k8sapp_openbao/tests +top_dir=./k8sapp_openbao +#parallel_class=True diff --git a/python3-k8sapp-openbao/k8sapp_openbao/LICENSE b/python3-k8sapp-openbao/k8sapp_openbao/LICENSE new file mode 100644 index 0000000..d6e2801 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Wind River Systems, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python3-k8sapp-openbao/k8sapp_openbao/README.rst b/python3-k8sapp-openbao/k8sapp_openbao/README.rst new file mode 100644 index 0000000..179de15 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/README.rst @@ -0,0 +1,7 @@ +k8sapp-openbao +=================== + +This project contains StarlingX Kubernetes application specific python plugins +for Openbao. These plugins are required to integrate the openbao authorization +application into the StarlingX application framework and to support the +various StarlingX deployments. diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/__init__.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/common/__init__.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/common/constants.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/common/constants.py new file mode 100644 index 0000000..099c3aa --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/common/constants.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Constants values for openbao helm""" + +# Helm: Supported charts: +# These values match the names in the chart package's Chart.yaml +HELM_APP_OPENBAO = 'openbao' +HELM_RELEASE_OPENBAO = 'stx-openbao' +HELM_CHART_OPENBAO = 'openbao' +HELM_RELEASE_OPENBAO_MANAGER = 'stx-openbao-manager' +HELM_CHART_OPENBAO_MANAGER = 'openbao-manager' +HELM_CHART_NS_OPENBAO = 'openbao' +HELM_OPENBAO_SERVER_POD = 'server' +HELM_OPENBAO_MANAGER_POD = 'manager' +HELM_OPENBAO_INJECTOR_POD = 'injector' + +HELM_CHART_COMPONENT_LABEL = 'app.starlingx.io/component' + +KEYSHARDS = 5 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/__init__.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/openbao.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/openbao.py new file mode 100644 index 0000000..34809e8 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/openbao.py @@ -0,0 +1,166 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Application helm class""" + +from k8sapp_openbao.common import constants as app_constants + +from oslo_log import log as logging + +from sysinv.common import constants +from sysinv.common import exception +from sysinv.common import kubernetes + +from sysinv.helm import base +from sysinv.helm import common + +from sysinv.db import api as dbapi + +import yaml + +LOG = logging.getLogger(__name__) + + +class OpenbaoHelm(base.FluxCDBaseHelm): + """Class to encapsulate helm operations for the openbao chart""" + + SUPPORTED_NAMESPACES = base.BaseHelm.SUPPORTED_NAMESPACES + \ + [common.HELM_NS_OPENBAO] + + SUPPORTED_APP_NAMESPACES = { + constants.HELM_APP_OPENBAO: + base.BaseHelm.SUPPORTED_NAMESPACES + [common.HELM_NS_OPENBAO], + } + + SUPPORTED_COMPONENT_OVERRIDES = ['application', 'platform'] + DEFAULT_AFFINITY = 'platform' + LABEL_PARAMETER = 'extraLabels' + + CHART = app_constants.HELM_CHART_OPENBAO + HELM_RELEASE = app_constants.HELM_RELEASE_OPENBAO + + def get_namespaces(self): + """Return the list of supported namespaces""" + return self.SUPPORTED_NAMESPACES + + def get_master_worker_host_count(self): + """Read the number of nodes with worker function""" + controller = len(self.dbapi.ihost_get_by_personality(constants.CONTROLLER)) + worker = len(self.dbapi.ihost_get_by_personality(constants.WORKER)) + return controller + worker + + def get_overrides(self, namespace=None): + """Return the system overrides""" + if self.get_master_worker_host_count() >= 3: + ha_replicas = 3 + else: + ha_replicas = 1 + + dbapi_instance = dbapi.get_instance() + + db_app = dbapi_instance.kube_app_get(app_constants.HELM_APP_OPENBAO) + + # User chart overrides + new_chart_overrides = self._get_helm_overrides( + dbapi_instance, + db_app, + app_constants.HELM_CHART_OPENBAO, + app_constants.HELM_CHART_NS_OPENBAO, + 'user_overrides') + + k8s_version = "" + + try: + kube = kubernetes.KubeOperator() + k8s_version = kube.kube_get_kubernetes_version() + except exception.KubeNotConfigured: + # Do not check for psp override if kubernetes is not configured yet + pass + + if (k8s_version >= "v1.25.1" + and new_chart_overrides + and "global" in new_chart_overrides.keys() + and "psp" in new_chart_overrides["global"].keys() + and "enable" in new_chart_overrides["global"]["psp"].keys() + and new_chart_overrides["global"]["psp"]["enable"] is True): + LOG.info("PSP must be disabled for kubernetes version 1.25 and onwards, " + "as the feature is depreciated. User helm override will be changed " + "so that global.psp.enabled is false") + new_chart_overrides["global"]["psp"]["enable"] = False + self._update_helm_overrides( + dbapi_instance, + db_app, + app_constants.HELM_CHART_OPENBAO, + app_constants.HELM_CHART_NS_OPENBAO, + 'user_overrides', + new_chart_overrides + ) + + user_chosen_affinity = new_chart_overrides.get( + app_constants.HELM_CHART_COMPONENT_LABEL) \ + if new_chart_overrides else None + + if user_chosen_affinity in self.SUPPORTED_COMPONENT_OVERRIDES: + affinity = user_chosen_affinity + else: + affinity = self.DEFAULT_AFFINITY + LOG.warn((f'User override for core affinity {user_chosen_affinity} ' + f'is invalid, using default of {self.DEFAULT_AFFINITY}')) + + overrides = { + common.HELM_NS_OPENBAO: { + app_constants.HELM_OPENBAO_SERVER_POD: { + 'ha': { + 'replicas': ha_replicas, + }, + self.LABEL_PARAMETER: { + app_constants.HELM_CHART_COMPONENT_LABEL: affinity + } + }, + app_constants.HELM_OPENBAO_INJECTOR_POD: { + self.LABEL_PARAMETER: { + app_constants.HELM_CHART_COMPONENT_LABEL: affinity + } + }, + } + } + + if namespace in self.SUPPORTED_NAMESPACES: + return overrides[namespace] + if namespace: + raise exception.InvalidHelmNamespace(chart=self.CHART, + namespace=namespace) + return overrides + + @staticmethod + def _get_helm_overrides(dbapi_instance, app, chart, namespace, + type_of_overrides): + """Helper function for querying helm overrides from db.""" + helm_overrides = {} + try: + helm_overrides = dbapi_instance.helm_override_get( + app_id=app.id, + name=chart, + namespace=namespace, + )[type_of_overrides] + + if isinstance(helm_overrides, str): + helm_overrides = yaml.safe_load(helm_overrides) + except exception.HelmOverrideNotFound: + LOG.debug("Overrides for this chart not found, nothing to be done.") + return helm_overrides + + @staticmethod + def _update_helm_overrides(dbapi_instance, app, chart, namespace, + type_of_overrides, value): + """Helper function for updating helm overrides to db.""" + helm_overrides = {type_of_overrides: yaml.safe_dump(value)} + dbapi_instance.helm_override_update( + app_id=app.id, + name=chart, + namespace=namespace, + values=helm_overrides + ) diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/openbao_manager.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/openbao_manager.py new file mode 100644 index 0000000..cc94c86 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/helm/openbao_manager.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Application helm class""" + +from k8sapp_openbao.common import constants as app_constants + +from oslo_log import log as logging + +from sysinv.common import constants +from sysinv.common import exception + +from sysinv.helm import base +from sysinv.helm import common + +from sysinv.db import api as dbapi + +import yaml + +LOG = logging.getLogger(__name__) + + +class OpenbaoManagerHelm(base.FluxCDBaseHelm): + """Class to encapsulate helm operations for the openbao manager chart""" + + SUPPORTED_NAMESPACES = base.BaseHelm.SUPPORTED_NAMESPACES + \ + [common.HELM_NS_OPENBAO] + + SUPPORTED_APP_NAMESPACES = { + constants.HELM_APP_OPENBAO: + base.BaseHelm.SUPPORTED_NAMESPACES + [common.HELM_NS_OPENBAO], + } + + SUPPORTED_COMPONENT_OVERRIDES = ['application', 'platform'] + DEFAULT_AFFINITY = 'platform' + LABEL_PARAMETER = 'extraLabels' + + CHART = app_constants.HELM_CHART_OPENBAO_MANAGER + HELM_RELEASE = app_constants.HELM_RELEASE_OPENBAO_MANAGER + + def execute_kustomize_updates(self, operator): + # On application load this chart is enabled. Only disable if + # specified by the user + if not self._is_enabled(operator.APP, self.CHART, + common.HELM_NS_OPENBAO): + operator.helm_release_resource_delete(self.CHART) + + def get_namespaces(self): + """Return the list of supported namespaces""" + return self.SUPPORTED_NAMESPACES + + def get_master_worker_host_count(self): + """Read the number of nodes with worker function""" + controller = len(self.dbapi.ihost_get_by_personality(constants.CONTROLLER)) + worker = len(self.dbapi.ihost_get_by_personality(constants.WORKER)) + return controller + worker + + def get_overrides(self, namespace=None): + """Return the system overrides""" + if self.get_master_worker_host_count() >= 3: + ha_replicas = 3 + else: + ha_replicas = 1 + + dbapi_instance = dbapi.get_instance() + + db_app = dbapi_instance.kube_app_get(app_constants.HELM_APP_OPENBAO) + + # User chart overrides + new_chart_overrides = self._get_helm_overrides( + dbapi_instance, + db_app, + app_constants.HELM_CHART_OPENBAO_MANAGER, + app_constants.HELM_CHART_NS_OPENBAO, + 'user_overrides') + + user_chosen_affinity = new_chart_overrides.get( + app_constants.HELM_CHART_COMPONENT_LABEL) \ + if new_chart_overrides else None + + if user_chosen_affinity in self.SUPPORTED_COMPONENT_OVERRIDES: + affinity = user_chosen_affinity + else: + affinity = self.DEFAULT_AFFINITY + LOG.warn((f'User override for core affinity {user_chosen_affinity} ' + f'is invalid, using default of {self.DEFAULT_AFFINITY}')) + + overrides = { + common.HELM_NS_OPENBAO: { + app_constants.HELM_OPENBAO_MANAGER_POD: { + 'ha': { + 'replicas': ha_replicas, + }, + self.LABEL_PARAMETER: { + app_constants.HELM_CHART_COMPONENT_LABEL: affinity + } + }, + } + } + + if namespace in self.SUPPORTED_NAMESPACES: + return overrides[namespace] + if namespace: + raise exception.InvalidHelmNamespace(chart=self.CHART, + namespace=namespace) + return overrides + + @staticmethod + def _get_helm_overrides(dbapi_instance, app, chart, namespace, + type_of_overrides): + """Helper function for querying helm overrides from db.""" + helm_overrides = {} + try: + helm_overrides = dbapi_instance.helm_override_get( + app_id=app.id, + name=chart, + namespace=namespace, + )[type_of_overrides] + + if isinstance(helm_overrides, str): + helm_overrides = yaml.safe_load(helm_overrides) + except exception.HelmOverrideNotFound: + LOG.debug("Overrides for this chart not found, nothing to be done.") + return helm_overrides diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/kustomize/__init__.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/kustomize/__init__.py new file mode 100644 index 0000000..06516bf --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/kustomize/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import yaml + + +class quoted_str(str): + pass + + +# force strings to be single-quoted to avoid interpretation as numeric values +def quoted_presenter(dumper, data): + return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style="'") + + +yaml.add_representer(quoted_str, quoted_presenter) diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/kustomize/kustomize_openbao.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/kustomize/kustomize_openbao.py new file mode 100644 index 0000000..7a36890 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/kustomize/kustomize_openbao.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# All Rights Reserved. +# + +""" System inventory Kustomization resource operator.""" + +from sysinv.common import constants +from sysinv.helm import kustomize_base as base + + +class OpenbaoFluxCDKustomizeOperator(base.FluxCDKustomizeOperator): + + APP = constants.HELM_APP_OPENBAO + + def platform_mode_kustomize_updates(self, dbapi, mode): + """ Update the top-level kustomization resource list + + Make changes to the top-level kustomization resource list based + on the platform mode + + :param dbapi: DB api object + :param mode: mode to control when to update the resource list + """ + pass diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/lifecycle/__init__.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/lifecycle/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/lifecycle/lifecycle_openbao.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/lifecycle/lifecycle_openbao.py new file mode 100644 index 0000000..fe1184f --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/lifecycle/lifecycle_openbao.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# All Rights Reserved. +# + +""" System inventory App lifecycle operator.""" + +from oslo_log import log as logging +from sysinv.helm import lifecycle_base as base + +LOG = logging.getLogger(__name__) + + +class OpenbaoAppLifecycleOperator(base.AppLifecycleOperator): + """Lifecycle operator for openbao application""" + + def app_lifecycle_actions(self, context, conductor_obj, app_op, app, hook_info): + """Perform lifecycle actions for an operation + + :param context: request context, can be None + :param conductor_obj: conductor object, can be None + :param app_op: AppOperator object + :param app: AppOperator.Application object + :param hook_info: LifecycleHookInfo object + + """ + LOG.info("lifecycle_type: {}, operation: {}, relative_timing: {}".format( + hook_info.lifecycle_type, + hook_info.operation, + hook_info.relative_timing)) + + super().app_lifecycle_actions(context, conductor_obj, app_op, + app, hook_info) diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/__init__.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/test_openbao.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/test_openbao.py new file mode 100644 index 0000000..1f0d249 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/test_openbao.py @@ -0,0 +1,49 @@ +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from k8sapp_openbao.common import constants as app_constants +from k8sapp_openbao.tests import test_plugins + +from sysinv.db import api as dbapi +from sysinv.helm import common + +from sysinv.tests.db import base as dbbase +from sysinv.tests.db import utils as dbutils +from sysinv.tests.helm import base + + +class OpenbaoTestCase(test_plugins.K8SAppOpenbaoAppMixin, + base.HelmTestCaseMixin): + + def setUp(self): + super(OpenbaoTestCase, self).setUp() + self.app = dbutils.create_test_app(name='openbao') + self.dbapi = dbapi.get_instance() + + +class OpenbaoIPv4ControllerHostTestCase(OpenbaoTestCase, + dbbase.ProvisionedControllerHostTestCase): + + def test_replicas(self): + overrides = self.operator.get_helm_chart_overrides( + app_constants.HELM_CHART_OPENBAO, + cnamespace=common.HELM_NS_OPENBAO) + + self.assertOverridesParameters(overrides, { + 'server': {'ha': {'replicas': 1}} + }) + + +class OpenbaoIPv6AIODuplexSystemTestCase(OpenbaoTestCase, + dbbase.BaseIPv6Mixin, + dbbase.ProvisionedAIODuplexSystemTestCase): + + def test_replicas(self): + overrides = self.operator.get_helm_chart_overrides( + app_constants.HELM_CHART_OPENBAO, + cnamespace=common.HELM_NS_OPENBAO) + + self.assertOverridesParameters(overrides, { + 'server': {'ha': {'replicas': 1}} + }) diff --git a/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/test_plugins.py b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/test_plugins.py new file mode 100644 index 0000000..8123cf1 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/k8sapp_openbao/tests/test_plugins.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sysinv.common import constants +from sysinv.tests.db import base as dbbase + + +class K8SAppOpenbaoAppMixin(object): + app_name = constants.HELM_APP_OPENBAO + path_name = app_name + '.tgz' + + def setUp(self): + super(K8SAppOpenbaoAppMixin, self).setUp() + + +# Test Configuration: +# - Controller +# - IPv6 +# - Ceph Storage +# - Openbao app +class K8sAppOpenbaoControllerTestCase(K8SAppOpenbaoAppMixin, + dbbase.BaseIPv6Mixin, + dbbase.BaseCephStorageBackendMixin, + dbbase.ControllerHostTestCase): + pass + + +# Test Configuration: +# - AIO +# - IPv4 +# - Ceph Storage +# - Openbao app +class K8SAppOpenbaoAIOTestCase(K8SAppOpenbaoAppMixin, + dbbase.BaseCephStorageBackendMixin, + dbbase.AIOSimplexHostTestCase): + pass diff --git a/python3-k8sapp-openbao/k8sapp_openbao/pylint.rc b/python3-k8sapp-openbao/k8sapp_openbao/pylint.rc new file mode 100644 index 0000000..d9e84e0 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/pylint.rc @@ -0,0 +1,336 @@ +[MASTER] +# Specify a configuration file. +rcfile=pylint.rc + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. Should be base names, not paths. +ignore= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist=lxml.etree,greenlet + + + +[MESSAGES CONTROL] +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). +# See "Messages Control" section of +# https://pylint.readthedocs.io/en/latest/user_guide +disable= + # C codes refer to Convention + C0103, # invalid-name + C0104, # disallowed-nameA + C0112, # empty-docstring + C0114, # missing-module-docstring + C0115, # missing-class-docstring + C0116, # missing-function-docstring + C0123, # unidiomatic-typecheck !!! + C0201, # consider-iterating-dictionary + C0202, # bad-classmethod-argument + C0206, # consider-using-dict-items + C0207, # use-maxsplit-arg + C0209, # consider-using-f-string + C0301, # line-too-long + C0302, # too-many-lines + C0325, # superfluous-parens + C0411, # wrong-import-order + C0412, # ungrouped-imports + C0413, # wrong-import-position + C0414, # useless-import-alias !!! + C0415, # import-outside-toplevel + C1802, # use-implicit-booleaness-not-len !!! + C2801, # unnecessary-dunder-call !!! + C3002, # unnecessary-direct-lambda-call !!! + # R codes refer to refactoring + R0022, # useless-option-value !!! + R0205, # useless-object-inheritance + R0402, # consider-using-from-import + R0901, # too-many-ancestors + R0902, # too-many-instance-attributes + R0903, # too-few-public-methods + R0904, # too-many-public-methods + R0911, # too-many-return-statements + R0912, # too-many-branches + R0913, # too-many-arguments + R0914, # too-many-locals + R0915, # too-many-statements + R0916, # too-many-boolean-expressions + R1702, # too-many-nested-blocks + R1703, # simplifiable-if-statement + R1704, # redefined-argument-from-local !!! + R1705, # no-else-return + R1707, # trailing-comma-tuple !!! + R1708, # stop-iteration-return !!! + R1710, # inconsistent-return-statements + R1711, # useless-return + R1714, # consider-using-in + R1717, # consider-using-dict-comprehension !!! + R1718, # consider-using-set-comprehension + R1719, # simplifiable-if-expression + R1720, # no-else-raise + R1721, # unnecessary-comprehension + R1722, # consider-using-sys-exit !!! + R1723, # no-else-break + R1724, # no-else-continue + R1725, # super-with-arguments + R1726, # simplifiable-condition !!! + R1728, # consider-using-generator + R1729, # use-a-generator + R1730, # consider-using-min-builtin !!! + R1731, # consider-using-max-builtin !!! + R1732, # consider-using-with + R1733, # unnecessary-dict-index-lookup !! + R1734, # use-list-literal + R1735, # use-dict-literal + # W codes are warnings + W0101, # unreachable + W0105, # pointless-string-statement + W0106, # expression-not-assigned + W0107, # unnecessary-pass + W0108, # unnecessary-lambda + W0109, # duplicate-key !!! + W0123, # eval-used + W0125, # using-constant-test !!! + W0133, # pointless-exception-statement !!! + W0143, # comparison-with-callable !!! + W0150, # lost-exception + W0201, # attribute-defined-outside-init + W0211, # bad-staticmethod-argument + W0212, # protected-access + W0221, # arguments-differ + W0223, # abstract-method + W0231, # super-init-not-called + W0235, # useless-super-delegation + W0237, # arguments-renamed !!! + W0311, # bad-indentation + W0402, # deprecated-module + W0404, # reimported + W0511, # fixme + W0602, # global-variable-not-assigned !!! + W0603, # global-statement + W0612, # unused-variable + W0613, # unused-argument + W0621, # redefined-outer-name + W0622, # redefined-builtin + W0631, # undefined-loop-variable + W0703, # broad-except (pylint 2.16 renamed to broad-except-caught) + W0706, # try-except-raise + W0707, # raise-missing-from + W0719, # broad-exception-raised + W1113, # keyword-arg-before-vararg + W1310, # format-string-without-interpolation !!! + W1401, # anomalous-backslash-in-string + W1406, # redundant-u-string-prefix + W1505, # deprecated-method + W1514, # unspecified-encoding + W3101, # missing-timeout + E0601, # used-before-assignment !!! + E0605, # invalid-all-format !!! + E1101, # no-member + E1111, # assignment-from-no-return + E1121, # too-many-function-args !!! + E1123, # unexpected-keyword-arg !!! + E1136, # unsubscriptable-object !!! + +[REPORTS] +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + + +[SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=85 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually 4 spaces or "\t" (1 tab). +indent-string=' ' + + +[TYPECHECK] +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules=distutils,eventlet.green.subprocess,six,six.moves + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +# pylint is confused by sqlalchemy Table, as well as sqlalchemy Enum types +# ie: (unprovisioned, identity) +# LookupDict in requests library confuses pylint +ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local, + Table, unprovisioned, identity, LookupDict + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[BASIC] +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + + +[MISCELLANEOUS] +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[VARIABLES] +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[IMPORTS] +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[DESIGN] +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[CLASSES] +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + + +[EXCEPTIONS] +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception diff --git a/python3-k8sapp-openbao/k8sapp_openbao/requirements.txt b/python3-k8sapp-openbao/k8sapp_openbao/requirements.txt new file mode 100644 index 0000000..8f225d8 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/requirements.txt @@ -0,0 +1,2 @@ +pbr>=2.0.0 +PyYAML>=3.10.0 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/setup.cfg b/python3-k8sapp-openbao/k8sapp_openbao/setup.cfg new file mode 100644 index 0000000..526efd4 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/setup.cfg @@ -0,0 +1,47 @@ +[metadata] +name = k8sapp-openbao +summary = StarlingX sysinv extensions for openbao +long_description = file: README.rst +long_description_content_type = text/x-rst +license = Apache 2.0 +author = StarlingX +author-email = starlingx-discuss@lists.starlingx.io +home-page = https://www.starlingx.io/ +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + +[files] +packages = + k8sapp_openbao + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[entry_points] +systemconfig.helm_applications = + openbao = systemconfig.helm_plugins.openbao + + +systemconfig.helm_plugins.openbao = + 001_openbao = k8sapp_openbao.helm.openbao:OpenbaoHelm + 002_openbao-manager = k8sapp_openbao.helm.openbao_manager:OpenbaoManagerHelm + +systemconfig.fluxcd.kustomize_ops = + openbao = k8sapp_openbao.kustomize.kustomize_openbao:OpenbaoFluxCDKustomizeOperator + +systemconfig.app_lifecycle = + openbao = k8sapp_openbao.lifecycle.lifecycle_openbao:OpenbaoAppLifecycleOperator + +[bdist_wheel] +universal = 1 diff --git a/python3-k8sapp-openbao/k8sapp_openbao/setup.py b/python3-k8sapp-openbao/k8sapp_openbao/setup.py new file mode 100644 index 0000000..598b345 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/setup.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import setuptools + + +setuptools.setup( + setup_requires=['pbr>=2.0.0'], + pbr=True) diff --git a/python3-k8sapp-openbao/k8sapp_openbao/test-requirements.txt b/python3-k8sapp-openbao/k8sapp_openbao/test-requirements.txt new file mode 100644 index 0000000..a140bcc --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/test-requirements.txt @@ -0,0 +1,20 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +hacking>=1.1.0,<=2.0.0 # Apache-2.0 +astroid +bandit<1.7.2;python_version>="3.0" +coverage>=3.6 +fixtures>=3.0.0 # Apache-2.0/BSD +mock>=2.0.0 # BSD +python-subunit>=0.0.18 +requests-mock>=0.6.0 # Apache-2.0 +sphinx +oslosphinx +oslotest>=3.2.0 # Apache-2.0 +stestr>=1.0.0 # Apache-2.0 +testrepository>=0.0.18 +testtools!=1.2.0,>=0.9.36 +isort<5;python_version>="3.0" +pylint +pycryptodomex diff --git a/python3-k8sapp-openbao/k8sapp_openbao/tox.ini b/python3-k8sapp-openbao/k8sapp_openbao/tox.ini new file mode 100644 index 0000000..f0de900 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/tox.ini @@ -0,0 +1,188 @@ +[tox] +envlist = flake8,py39,pylint,metadata +minversion = 1.6 +skipsdist = True + +# tox does not work if the path to the workdir is too long, so move it to /tmp +# tox 3.1.0 adds TOX_LIMITED_SHEBANG +toxworkdir = /tmp/{env:USER}_k8sopenbaotox +stxdir = {toxinidir}/../../.. +distshare={toxworkdir}/.tox/distshare + +[testenv] +basepython = python3.9 +usedevelop = True + +# tox is silly... these need to be separated by a newline.... +allowlist_externals = bash + find + echo + +install_command = pip install -v -v -v \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ + {opts} {packages} + +# Note the hash seed is set to 0 until can be tested with a +# random hash seed successfully. +setenv = VIRTUAL_ENV={envdir} + PYTHONHASHSEED=0 + PIP_RESOLVER_DEBUG=1 + PYTHONDONTWRITEBYTECODE=1 + OS_TEST_PATH=./k8sapp_openbao/tests + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C + EVENTS_YAML=./k8sapp_openbao/tests/events_for_testing.yaml + SYSINV_TEST_ENV=True + TOX_WORK_DIR={toxworkdir} + PYLINTHOME={toxworkdir} + +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + -e{[tox]stxdir}/config/sysinv/sysinv/sysinv + -e{[tox]stxdir}/config/tsconfig/tsconfig + -e{[tox]stxdir}/fault/fm-api/source + -e{[tox]stxdir}/fault/python-fmclient/fmclient + -e{[tox]stxdir}/update/sw-patch/cgcs-patch + -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient + +commands = + find . -type f -name "*.pyc" -delete + +[flake8] +# H series are hacking +# H101 is TODO +# H102 is apache license +# H104 file contains only comments (ie: license) +# H105 author tags +# H306 imports not in alphabetical order +# H401 docstring should not start with a space +# H403 multi line docstrings should end on a new line +# H404 multi line docstring should start without a leading new line +# H405 multi line docstring summary not separated with an empty line +# H701 Empty localization string +# H702 Formatting operation should be outside of localization method call +# H703 Multiple positional placeholders + +# B series are bugbear +# B006 Do not use mutable data structures for argument defaults. Needs to be FIXED. +# B007 Loop control variable not used within the loop body. +# B009 Do not call getattr with a constant attribute value +# B010 Do not call setattr with a constant attribute value +# B012 return/continue/break inside finally blocks cause exceptions to be silenced +# B014 Redundant exception types +# B301 Python 3 does not include `.iter*` methods on dictionaries. (this should be suppressed on a per line basis) + +# W series are warnings +# W503 line break before binary operator +# W504 line break after binary operator +# W605 invalid escape sequence + +# E series are pep8 +# E117 over-indented +# E126 continuation line over-indented for hanging indent +# E127 continuation line over-indented for visual indent +# E128 continuation line under-indented for visual indent +# E402 module level import not at top of file +# E741 ambiguous variable name + +ignore = H101,H102,H104,H105,H306,H401,H403,H404,H405,H701,H702,H703, + B006,B007,B009,B010,B012,B014,B301 + W503,W504,W605, + E117,E126,E127,E128,E402,E741 +exclude = build,dist,tools,.eggs +max-line-length=120 + +[testenv:flake8] +deps = -r{toxinidir}/test-requirements.txt +commands = + flake8 {posargs} ./k8sapp_openbao + +[testenv:py39] +commands = + stestr run {posargs} + stestr slowest + +[testenv:pep8] +# testenv:flake8 clone +deps = -r{toxinidir}/test-requirements.txt +commands = {[testenv:flake8]commands} + +[testenv:venv] +commands = {posargs} + +[bandit] +# The following bandit tests are being skipped: +# B101: Test for use of assert +# B103: Test for setting permissive file permissions +# B104: Test for binding to all interfaces +# B105: Test for use of hard-coded password strings +# B108: Test for insecure usage of tmp file/directory +# B110: Try, Except, Pass detected. +# B303: Use of insecure MD2, MD4, MD5, or SHA1 hash function. +# B307: Blacklisted call to eval. +# B310: Audit url open for permitted schemes +# B311: Standard pseudo-random generators are not suitable for security/cryptographic purposes +# B314: Blacklisted calls to xml.etree.ElementTree +# B318: Blacklisted calls to xml.dom.minidom +# B320: Blacklisted calls to lxml.etree +# B404: Import of subprocess module +# B405: import xml.etree +# B408: import xml.minidom +# B410: import lxml +# B506: Test for use of yaml load +# B602: Test for use of popen with shell equals true +# B603: Test for use of subprocess without shell equals true +# B604: Test for any function with shell equals true +# B605: Test for starting a process with a shell +# B607: Test for starting a process with a partial path +# B608: Possible SQL injection vector through string-based query +# +# Note: 'skips' entry cannot be split across multiple lines +# +skips = B101,B103,B104,B105,B108,B110,B303,B307,B310,B311,B314,B318,B320,B404,B405,B408,B410,B506,B602,B603,B604,B605,B607,B608 +exclude = tests + +[testenv:bandit] +deps = -r{toxinidir}/test-requirements.txt +commands = bandit --ini tox.ini -n 5 -r k8sapp_openbao + +[testenv:pylint] +install_command = pip install -v -v -v \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ + {opts} {packages} +commands = + pylint {posargs} k8sapp_openbao --rcfile=./pylint.rc + +[testenv:cover] +# not sure is passenv is still needed +passenv = CURL_CA_BUNDLE +deps = {[testenv]deps} +setenv = {[testenv]setenv} + PYTHON=coverage run --parallel-mode + +commands = + {[testenv]commands} + coverage erase + stestr run {posargs} + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report + +[testenv:pip-missing-reqs] +# do not install test-requirements as that will pollute the virtualenv for +# determining missing packages +# this also means that pip-missing-reqs must be installed separately, outside +# of the requirements.txt files +deps = pip_missing_reqs + -rrequirements.txt +commands=pip-missing-reqs -d --ignore-file=/k8sapp_openbao/tests k8sapp_openbao + +[testenv:metadata] +install_command = pip install -v -v -v \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ + {opts} {packages} +# Pass top level app folder to 'sysinv-app tox' command. +commands = + bash -c "echo $(dirname $(dirname $(pwd))) | xargs -n 1 sysinv-app tox" \ No newline at end of file diff --git a/python3-k8sapp-openbao/k8sapp_openbao/upper-constraints.txt b/python3-k8sapp-openbao/k8sapp_openbao/upper-constraints.txt new file mode 100644 index 0000000..9c30188 --- /dev/null +++ b/python3-k8sapp-openbao/k8sapp_openbao/upper-constraints.txt @@ -0,0 +1 @@ +# Override upstream constraints based on StarlingX load diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c01ade2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +# Nothing diff --git a/stx-openbao-helm/debian/deb_folder/changelog b/stx-openbao-helm/debian/deb_folder/changelog new file mode 100644 index 0000000..ac6a5ee --- /dev/null +++ b/stx-openbao-helm/debian/deb_folder/changelog @@ -0,0 +1,5 @@ +stx-openbao-helm (1.0-1) unstable; urgency=medium + + * Initial release. + + -- Tae Park Thu, 10 Oct 2024 19:34:18 -0400 diff --git a/stx-openbao-helm/debian/deb_folder/control b/stx-openbao-helm/debian/deb_folder/control new file mode 100644 index 0000000..78eeaca --- /dev/null +++ b/stx-openbao-helm/debian/deb_folder/control @@ -0,0 +1,19 @@ +Source: stx-openbao-helm +Section: libs +Priority: optional +Maintainer: StarlingX Developers +Build-Depends: debhelper-compat (= 13), + openbao-helm, + openbao-manager-helm, + helm, + python3-k8sapp-openbao-wheels, + build-info +Standards-Version: 4.5.1 +Homepage: https://www.starlingx.io + +Package: stx-openbao-helm +Section: libs +Architecture: any +Depends: ${misc:Depends} +Description: StarlingX Openbao FluxCD Helm Charts + This package contains FluxCD helm charts for the openbao application. diff --git a/stx-openbao-helm/debian/deb_folder/copyright b/stx-openbao-helm/debian/deb_folder/copyright new file mode 100644 index 0000000..20e3194 --- /dev/null +++ b/stx-openbao-helm/debian/deb_folder/copyright @@ -0,0 +1,41 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: stx-openbao-helm +Source: https://opendev.org/starlingx/app-openbao/ + +Files: * +Copyright: (c) 2013-2021 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. + +# If you want to use GPL v2 or later for the /debian/* files use +# the following clauses, or change it to suit. Delete these two lines +Files: debian/* +Copyright: 2021 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. diff --git a/stx-openbao-helm/debian/deb_folder/rules b/stx-openbao-helm/debian/deb_folder/rules new file mode 100755 index 0000000..156b5c2 --- /dev/null +++ b/stx-openbao-helm/debian/deb_folder/rules @@ -0,0 +1,62 @@ +#!/usr/bin/make -f +# export DH_VERBOSE = 1 + +export ROOT = debian/tmp +export APP_FOLDER = $(ROOT)/usr/local/share/applications/helm + +export DEB_VERSION = $(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') +export RELEASE = $(shell cat /etc/build.info | grep SW_VERSION | cut -d'"' -f2) +export REVISION = $(shell echo $(DEB_VERSION) | cut -f 4 -d '.') + +export APP_NAME = openbao +export APP_VERSION = $(RELEASE)-$(REVISION) +export APP_TARBALL = $(APP_NAME)-$(APP_VERSION).tgz +export HELM_REPO = stx-platform +export STAGING = staging + +%: + dh $@ + +override_dh_auto_build: + # Setup the staging directory. + mkdir -p $(STAGING) + cp files/metadata.yaml $(STAGING) + cp -Rv fluxcd-manifests $(STAGING)/ + mkdir -p $(STAGING)/charts + cp /usr/lib/helm/openbao*.tgz $(STAGING)/charts + + # Adjust the helmrelease yamls based on the chart versions + for c in $(STAGING)/charts/*; do \ + chart=$$(basename $$c .tgz); \ + chart_name=$${chart%-*}; \ + chart_version=$${chart##*-}; \ + echo "Found $$chart; name: $$chart_name, version: $$chart_version"; \ + chart_manifest=$$(find $(STAGING)/fluxcd-manifests/$$chart_name -name helmrelease.yaml -exec grep -q $$chart_name {} \; -print); \ + echo "Updating manifest: $$chart_manifest"; \ + sed -i "s/REPLACE_HELM_CHART_VERSION/$$chart_version/g" $$chart_manifest; \ + grep version $$chart_manifest; \ + done + + # Populate metadata. + sed -i 's/APP_REPLACE_NAME/$(APP_NAME)/g' $(STAGING)/metadata.yaml + sed -i 's/APP_REPLACE_VERSION/$(APP_VERSION)/g' $(STAGING)/metadata.yaml + sed -i 's/HELM_REPLACE_REPO/$(HELM_REPO)/g' $(STAGING)/metadata.yaml + + # Copy the plugins: installed in the buildroot + mkdir -p $(STAGING)/plugins + cp /plugins/*.whl $(STAGING)/plugins + + # calculate checksum of all files in staging for the app + cd $(STAGING) && find . -type f ! -name '*.md5' -print0 | xargs -0 md5sum > checksum.md5 + # package the app + tar -zcf $(APP_TARBALL) -C $(STAGING)/ . + + # Cleanup staging + rm -rf $(STAGING) + +override_dh_auto_install: + # Install the app tar file. + install -d -m 755 $(APP_FOLDER) + install -p -D -m 755 $(APP_TARBALL) $(APP_FOLDER) + +override_dh_usrlocal: diff --git a/stx-openbao-helm/debian/deb_folder/source/format b/stx-openbao-helm/debian/deb_folder/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/stx-openbao-helm/debian/deb_folder/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/stx-openbao-helm/debian/deb_folder/stx-openbao-helm.install b/stx-openbao-helm/debian/deb_folder/stx-openbao-helm.install new file mode 100644 index 0000000..1b47c6e --- /dev/null +++ b/stx-openbao-helm/debian/deb_folder/stx-openbao-helm.install @@ -0,0 +1 @@ +usr/local/share/applications/helm/* diff --git a/stx-openbao-helm/debian/meta_data.yaml b/stx-openbao-helm/debian/meta_data.yaml new file mode 100644 index 0000000..3fba632 --- /dev/null +++ b/stx-openbao-helm/debian/meta_data.yaml @@ -0,0 +1,9 @@ +--- +debname: stx-openbao-helm +debver: 1.0-1 +src_path: stx-openbao-helm +revision: + dist: $STX_DIST + SRC_GITREVCOUNT: + SRC_DIR: ${MY_REPO}/stx/app-openbao + SRC_BASE_SRCREV: 0144b018a6a592860dace539e9fb937af7b2d26f diff --git a/stx-openbao-helm/stx-openbao-helm/README b/stx-openbao-helm/stx-openbao-helm/README new file mode 100644 index 0000000..99b2247 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/README @@ -0,0 +1,4 @@ +This directory contains all fluxCD manifest required in the configuration +of the Openbao application for StarlingX. This includes config for the +upstream Openbao helm chart, Openbao manager chart, and the python sysinv +integration for Openbao. diff --git a/stx-openbao-helm/stx-openbao-helm/files/metadata.yaml b/stx-openbao-helm/stx-openbao-helm/files/metadata.yaml new file mode 100644 index 0000000..628b0c7 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/files/metadata.yaml @@ -0,0 +1,17 @@ +app_name: APP_REPLACE_NAME +app_version: APP_REPLACE_VERSION +helm_repo: HELM_REPLACE_REPO + +maintain_user_overrides: true + +upgrades: + auto_update: true + +supported_k8s_version: + minimum: 1.24.4 + +behavior: + platform_managed_app: yes + evaluate_reapply: + after: + - platform-integ-apps diff --git a/stx-openbao-helm/stx-openbao-helm/files/repositories.yaml b/stx-openbao-helm/stx-openbao-helm/files/repositories.yaml new file mode 100644 index 0000000..e613b63 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/files/repositories.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +generated: 2019-01-02T15:19:36.215111369-06:00 +repositories: +- caFile: "" + cache: /builddir/.helm/repository/cache/local-index.yaml + certFile: "" + keyFile: "" + name: local + password: "" + url: http://127.0.0.1:8879/charts + username: "" + diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/helmrepository.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/helmrepository.yaml new file mode 100644 index 0000000..d20312c --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/helmrepository.yaml @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: stx-platform +spec: + url: http://192.168.206.1:8080/helm_charts/stx-platform + interval: 1m diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/kustomization.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/kustomization.yaml new file mode 100644 index 0000000..58eb3ee --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/kustomization.yaml @@ -0,0 +1,8 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +resources: + - helmrepository.yaml diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/namespace.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/namespace.yaml new file mode 100644 index 0000000..09e271b --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/base/namespace.yaml @@ -0,0 +1,10 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +apiVersion: v1 +kind: Namespace +metadata: + name: openbao diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/kustomization.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/kustomization.yaml new file mode 100644 index 0000000..940b4ab --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/kustomization.yaml @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: openbao +resources: + - base + - openbao + - openbao-manager diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/helmrelease.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/helmrelease.yaml new file mode 100644 index 0000000..7820175 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/helmrelease.yaml @@ -0,0 +1,36 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +apiVersion: "helm.toolkit.fluxcd.io/v2" +kind: HelmRelease +metadata: + name: openbao-manager + labels: + chart_group: openbao +spec: + releaseName: stx-openbao-manager + chart: + spec: + chart: openbao-manager + version: REPLACE_HELM_CHART_VERSION + sourceRef: + kind: HelmRepository + name: stx-platform + interval: 1m + timeout: 30m + test: + enable: false + install: + disableHooks: false + upgrade: + disableHooks: false + valuesFrom: + - kind: Secret + name: openbao-manager-static-overrides + valuesKey: openbao-manager-static-overrides.yaml + - kind: Secret + name: openbao-manager-system-overrides + valuesKey: openbao-manager-system-overrides.yaml diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/kustomization.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/kustomization.yaml new file mode 100644 index 0000000..64276f3 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/kustomization.yaml @@ -0,0 +1,18 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +namespace: openbao +resources: + - helmrelease.yaml +secretGenerator: + - name: openbao-manager-static-overrides + files: + - openbao-manager-static-overrides.yaml + - name: openbao-manager-system-overrides + files: + - openbao-manager-system-overrides.yaml +generatorOptions: + disableNameSuffixHash: true diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/openbao-manager-static-overrides.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/openbao-manager-static-overrides.yaml new file mode 100644 index 0000000..233d9d0 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/openbao-manager-static-overrides.yaml @@ -0,0 +1,24 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +manager: + image: + repository: starlingx/stx-vault-manager + tag: stx.10.0-v1.29.6-1 + tolerations: | + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + unsealWaitIntervals: 0 + imagePullSecrets: + - name: default-registry-key + livenessProbe: + initialDelaySeconds: 31 + periodSeconds: 23 + timeoutSeconds: 13 diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/openbao-manager-system-overrides.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/openbao-manager-system-overrides.yaml new file mode 100644 index 0000000..78c027e --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao-manager/openbao-manager-system-overrides.yaml @@ -0,0 +1,6 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/helmrelease.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/helmrelease.yaml new file mode 100644 index 0000000..e130c8c --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/helmrelease.yaml @@ -0,0 +1,36 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +apiVersion: "helm.toolkit.fluxcd.io/v2" +kind: HelmRelease +metadata: + name: openbao + labels: + chart_group: openbao +spec: + releaseName: stx-openbao + chart: + spec: + chart: openbao + version: REPLACE_HELM_CHART_VERSION + sourceRef: + kind: HelmRepository + name: stx-platform + interval: 1m + timeout: 30m + test: + enable: false + install: + disableHooks: false + upgrade: + disableHooks: false + valuesFrom: + - kind: Secret + name: openbao-static-overrides + valuesKey: openbao-static-overrides.yaml + - kind: Secret + name: openbao-system-overrides + valuesKey: openbao-system-overrides.yaml diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/kustomization.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/kustomization.yaml new file mode 100644 index 0000000..b38a2f9 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/kustomization.yaml @@ -0,0 +1,18 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +namespace: openbao +resources: + - helmrelease.yaml +secretGenerator: + - name: openbao-static-overrides + files: + - openbao-static-overrides.yaml + - name: openbao-system-overrides + files: + - openbao-system-overrides.yaml +generatorOptions: + disableNameSuffixHash: true diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/openbao-static-overrides.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/openbao-static-overrides.yaml new file mode 100644 index 0000000..99252d1 --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/openbao-static-overrides.yaml @@ -0,0 +1,117 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +global: + enabled: true + tlsDisable: false + imagePullSecrets: + - name: default-registry-key +injector: + enabled: true + nodeSelector: | + node-role.kubernetes.io/control-plane: "" + affinity: null + strategy: + rollingUpdate: + maxUnavailable: 100% + image: + registry: docker.io + repository: hashicorp/vault-k8s + tag: 1.2.1 + agentImage: + # Add yaml compatible with starlingx platform image pull, and + # service-parameter registry overrides. This will pull from + # private or public registry into registry.local. docker.io + # registry is assumed when omitted: + image: + registry: quay.io + repository: openbao/openbao + tag: 2.1.0 + # Set the openbao yaml to refer to registry.local pulled as above + registry: registry.local:9001/quay.io + repository: openbao/openbao + tag: 2.1.0 + tolerations: | + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + livenessProbe: + initialDelaySeconds: 11 + periodSeconds: 11 + timeoutSeconds: 7 +server: + affinity: | + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: {{ template "openbao.name" . }} + app.kubernetes.io/instance: "{{ .Release.Name }}" + component: server + topologyKey: kubernetes.io/hostname + image: + registry: quay.io + repository: openbao/openbao + tag: 2.1.0 + tolerations: | + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + auditStorage: + enabled: true + size: 10Gi + ha: + enabled: true + replicas: 3 + raft: + enabled: true + config: | + ui = false + + listener "tcp" { + tls_disable = 0 + address = "[::]:8200" + cluster_address = "[::]:8201" + tls_cert_file = "/openbao/userconfig/openbao-server-tls/tls.crt" + tls_key_file = "/openbao/userconfig/openbao-server-tls/tls.key" + tls_client_ca_file = "/openbao/userconfig/openbao-server-tls/ca.crt" + } + + storage "raft" { + path = "/openbao/data" + } + + service_registration "kubernetes" {} + extraLabels: + app: openbao + extraEnvironmentVars: + VAULT_CACERT: /openbao/userconfig/openbao-server-tls/ca.crt + volumes: + - name: openbao-server-tls + secret: + secretName: openbao-server-tls + volumeMounts: + - name: openbao-server-tls + readOnly: true + mountPath: /openbao/userconfig/openbao-server-tls + readinessProbe: + initialDelaySeconds: 25 +csi: + image: + registry: "docker.io" + repository: "hashicorp/vault-csi-provider" + tag: "1.4.0" + agent: + image: + registry: "quay.io" + repository: "openbao/openbao" + tag: "2.1.0" diff --git a/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/openbao-system-overrides.yaml b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/openbao-system-overrides.yaml new file mode 100644 index 0000000..78c027e --- /dev/null +++ b/stx-openbao-helm/stx-openbao-helm/fluxcd-manifests/openbao/openbao-system-overrides.yaml @@ -0,0 +1,6 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..8ae3e22 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,3 @@ +# hacking pulls in flake8 +hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 +bashate >= 0.2 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..feb7435 --- /dev/null +++ b/tox.ini @@ -0,0 +1,54 @@ +[tox] +envlist = linters +minversion = 2.9 +skipsdist = True +sitepackages=False + +[testenv] +basepython = python3 +install_command = pip install -U \ + {opts} {packages} \ + -c{env:TOX_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} +setenv = + VIRTUAL_ENV={envdir} + OS_STDOUT_CAPTURE=1 + OS_STDERR_CAPTURE=1 + OS_DEBUG=1 + OS_LOG_CAPTURE=1 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +allowlist_externals = + bash + +[testenv:flake8] +basepython = python3 +description = Dummy environment to allow flake8 to be run in subdir tox + +[testenv:pylint] +basepython = python3 +description = Dummy environment to allow pylint to be run in subdir tox + +[testenv:metadata] +basepython = python3 +description = Dummy environment to allow sysinv-app to be run in subdir tox + +[testenv:bandit] +basepython = python3 +description = Dummy environment to allow bandit to be run in subdir tox + +[testenv:bashate] +# Treat all E* codes as Errors rather than warnings using: -e 'E*' +commands = + bash -c "find {toxinidir} \ + -not \( -type d -name .?\* -prune \) \ + -type f \ + -not -name \*~ \ + -not -name \*.md \ + -name \*.sh \ + -print0 | xargs -r -n 1 -0 bashate -v \ + -e 'E*'" + +[testenv:linters] +commands = + {[testenv:bashate]commands}