starlingx/update
Code Issues Proposed changes

Merge "Activate-rollback implementation"

Browse Source
This commit is contained in:
Zuul 2024-06-24 19:10:59 +00:00 committed by Gerrit Code Review
commit 7847f7087e
12 changed files with 187 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
software-client/software_client/v1/deploy.py
View File

@ -94,6 +94,15 @@ class DeployManager(base.Manager):
return self._create(path, body={})
def activate_rollback(self, args):
# Ignore interrupts during this function
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Issue deploy_start request
path = "/v1/deploy/activate_rollback"
return self._create(path, body={})
def complete(self, args):
# Ignore interrupts during this function
signal.signal(signal.SIGINT, signal.SIG_IGN)

1
software-client/software_client/v1/deploy_cmd.py
View File

@ -19,6 +19,7 @@ DEPLOY_COMMAND_MODULES = [
# - host
# - abort
# - activate
# - activate-rollback
# - complete
# - delete
# non root/sudo users can run:

11
software-client/software_client/v1/deploy_shell.py
View File

@ -175,6 +175,17 @@ def do_activate(cc, args):
return utils.check_rc(resp, data)
def do_activate_rollback(cc, args):
"""Rolls back the activate of software deployment"""
resp, data = cc.deploy.activate_rollback(args)
if args.debug:
utils.print_result_debug(resp, data)
utils.display_info(resp)
return utils.check_rc(resp, data)
def do_complete(cc, args):
"""Complete the software deployment"""
resp, data = cc.deploy.complete(args)

1
software/setup.cfg
View File

@ -38,6 +38,7 @@ console_scripts =
software-migrate = software.utilities.migrate:migrate
software-deploy-update = software.utilities.update_deploy_state:update_state
software-deploy-activate = software.utilities.activate:activate
software-deploy-activate-rollback = software.utilities.activate_rollback:activate_rollback
[wheel]

9
software/software/api/controllers/v1/deploy.py
View File

@ -22,6 +22,7 @@ class DeployController(RestController):
_custom_actions = {
'abort': ['POST'],
'activate': ['POST'],
'activate_rollback': ['POST'],
'precheck': ['POST'],
'start': ['POST'],
'complete': ['POST'],
@ -45,6 +46,14 @@ class DeployController(RestController):
sc.software_sync()
return result
@expose(method='POST', template='json')
def activate_rollback(self):
reload_release_data()
result = sc.software_deploy_activate_rollback_api()
sc.software_sync()
return result
@expose(method='POST', template='json')
def complete(self):
reload_release_data()

15
software/software/deploy_state.py
View File

@ -46,14 +46,16 @@ deploy_state_transition = {
DEPLOY_STATES.ACTIVATE: [DEPLOY_STATES.ACTIVATE_DONE, DEPLOY_STATES.ACTIVATE_FAILED],
DEPLOY_STATES.ACTIVATE_FAILED: [DEPLOY_STATES.ACTIVATE, # deploy activate is reentrant
DEPLOY_STATES.ACTIVATE_ROLLBACK],
DEPLOY_STATES.ACTIVATE_DONE: [DEPLOY_STATES.COMPLETED, DEPLOY_STATES.ACTIVATE_ROLLBACK],
DEPLOY_STATES.ACTIVATE_DONE: [DEPLOY_STATES.COMPLETED, DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING],
# deploy activate rollback
DEPLOY_STATES.ACTIVATE_ROLLBACK: [DEPLOY_STATES.HOST_ROLLBACK, DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED],
DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED: [DEPLOY_STATES.ACTIVATE_ROLLBACK], # deploy host rollback is reentrant
DEPLOY_STATES.ACTIVATE_ROLLBACK: [DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE, DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED],
DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE: [DEPLOY_STATES.HOST_ROLLBACK],
DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED: [DEPLOY_STATES.ACTIVATE_ROLLBACK], # deploy activate rollback is reentrant
DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING: [DEPLOY_STATES.ACTIVATE_ROLLBACK],
# deploy complete
DEPLOY_STATES.COMPLETED: [DEPLOY_STATES.ACTIVATE_ROLLBACK, None]
DEPLOY_STATES.COMPLETED: [DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING, None]
}
@ -181,7 +183,7 @@ class DeployState(object):
# host rollback, if post-activate then go to activate rollback
state = DeployState.get_deploy_state()
if state in [DEPLOY_STATES.ACTIVATE_DONE, DEPLOY_STATES.ACTIVATE_FAILED, DEPLOY_STATES.COMPLETED]:
self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK)
self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING)
else:
self.transform(DEPLOY_STATES.HOST_ROLLBACK)
@ -212,6 +214,9 @@ class DeployState(object):
def activate_rollback(self):
self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK)
def activate_rollback_done(self):
self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE)
def activate_rollback_failed(self):
self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED)

65
software/software/software_controller.py
View File

@ -746,7 +746,7 @@ class SWMessageDeployStateChanged(messages.PatchMessage):
self.valid = True
self.agent = None
valid_agents = ['deploy-start', 'deploy-activate']
valid_agents = ['deploy-start', 'deploy-activate', 'deploy-activate-rollback']
if 'agent' in data:
self.agent = data['agent']
else:
@ -762,7 +762,9 @@ class SWMessageDeployStateChanged(messages.PatchMessage):
DEPLOY_STATES.START_DONE.value: DEPLOY_STATES.START_DONE,
DEPLOY_STATES.START_FAILED.value: DEPLOY_STATES.START_FAILED,
DEPLOY_STATES.ACTIVATE_FAILED.value: DEPLOY_STATES.ACTIVATE_FAILED,
DEPLOY_STATES.ACTIVATE_DONE.value: DEPLOY_STATES.ACTIVATE_DONE
DEPLOY_STATES.ACTIVATE_DONE.value: DEPLOY_STATES.ACTIVATE_DONE,
DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE.value: DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE,
DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED.value: DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED
}
if 'deploy-state' in data and data['deploy-state']:
deploy_state = data['deploy-state']
@ -2413,7 +2415,9 @@ class PatchController(PatchService):
DEPLOY_STATES.START_DONE: deploy_state.start_done,
DEPLOY_STATES.START_FAILED: deploy_state.start_failed,
DEPLOY_STATES.ACTIVATE_DONE: deploy_state.activate_done,
DEPLOY_STATES.ACTIVATE_FAILED: deploy_state.activate_failed
DEPLOY_STATES.ACTIVATE_FAILED: deploy_state.activate_failed,
DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE: deploy_state.activate_rollback_done,
DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED: deploy_state.activate_rollback_failed
}
if new_state in state_event:
state_event[new_state]()
@ -3013,6 +3017,61 @@ class PatchController(PatchService):
return dict(info=msg_info, warning=msg_warning, error=msg_error)
def _activate_rollback_major_release(self, deploy):
cmd_path = "/usr/bin/software-deploy-activate-rollback"
from_release = utils.get_major_release_version(deploy.get("from_release"))
to_release = utils.get_major_release_version(deploy.get("to_release"))
upgrade_activate_rollback_cmd = [cmd_path, from_release, to_release]
try:
LOG.info("starting subprocess %s" % ' '.join(upgrade_activate_rollback_cmd))
subprocess.Popen(' '.join(upgrade_activate_rollback_cmd), start_new_session=True, shell=True)
LOG.info("subprocess started")
except subprocess.SubprocessError as e:
LOG.error("Failed to start command: %s. Error %s" % (' '.join(upgrade_activate_rollback_cmd), e))
raise
def _activate_rollback_patching_release(self):
deploy_state = DeployState.get_instance()
# patching release activate-rollback operations go here
deploy_state.activate_rollback_done()
def _activate_rollback(self):
deploy = self.db_api_instance.get_current_deploy()
if not deploy:
msg = "Deployment is missing unexpectedly"
raise InvalidOperation(msg)
deploying = ReleaseState(release_state=states.DEPLOYING)
if deploying.is_major_release_deployment():
self._activate_rollback_major_release(deploy)
else:
self._activate_rollback_patching_release()
@require_deploy_state([DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING, DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED],
"Activate-rollback deployment only when current deployment state is {require_states}")
def software_deploy_activate_rollback_api(self) -> dict:
"""
Rolls back activates the deployment associated with the release
:return: dict of info, warning and error messages
"""
msg_info = ""
msg_warning = ""
msg_error = ""
deploy_state = DeployState.get_instance()
deploy_state.activate_rollback()
try:
self._activate_rollback()
msg_info = "Deploy activate-rollback has started"
except Exception:
deploy_state.activate_rollback_failed()
raise
return dict(info=msg_info, warning=msg_warning, error=msg_error)
def software_deploy_show_api(self, from_release=None, to_release=None):
# Retrieve deploy state from db
if from_release and to_release:

2
software/software/states.py
View File

@ -119,7 +119,9 @@ class DEPLOY_STATES(Enum):
ACTIVATE_FAILED = 'activate-failed'
ACTIVATE_ROLLBACK = 'activate-rollback'
ACTIVATE_ROLLBACK_DONE = 'activate-rollback-done'
ACTIVATE_ROLLBACK_FAILED = 'activate-rollback-failed'
ACTIVATE_ROLLBACK_PENDING = 'activate-rollback-pending'
COMPLETED = 'completed'

5
software/software/utilities/activate.py
View File

@ -9,6 +9,7 @@ from oslo_log import log
from software.states import DEPLOY_STATES
from software.utilities.update_deploy_state import update_deploy_state
from software.utilities.utils import configure_logging
from software.utilities.utils import execute_migration_scripts
from software.utilities.utils import ACTION_ACTIVATE
@ -38,8 +39,8 @@ def do_activate(from_release, to_release):
def activate():
# this is the entry point to start data migration
# this is the entry point to start activate
configure_logging()
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("from_release",

59
software/software/utilities/activate_rollback.py Normal file
View File

@ -0,0 +1,59 @@
#
# Copyright (c) 2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import argparse
from oslo_log import log
from software.states import DEPLOY_STATES
from software.utilities.update_deploy_state import update_deploy_state
from software.utilities.utils import configure_logging
from software.utilities.utils import execute_migration_scripts
from software.utilities.utils import ACTION_ACTIVATE_ROLLBACK
LOG = log.getLogger(__name__)
def do_activate_rollback(from_release, to_release):
agent = 'deploy-activate-rollback'
res = True
state = DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE.value
try:
execute_migration_scripts(from_release, to_release, ACTION_ACTIVATE_ROLLBACK)
except Exception:
state = DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED.value
res = False
finally:
try:
update_deploy_state(agent, deploy_state=state)
if res:
LOG.info("Deploy activate-rollback completed successfully")
else:
LOG.error("Deploy activate-rollback failed")
except Exception as err:
LOG.error("Update deploy state activate-rollback failed: %s" % err)
res = False
return res
def activate_rollback():
# this is the entry point to start activate-rollback
configure_logging()
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("from_release",
default=False,
help="From release")
parser.add_argument("to_release",
default=False,
help="To release")
args = parser.parse_args()
if do_activate_rollback(args.from_release, args.to_release):
exit(0)
else:
exit(1)

7
software/software/utilities/migrate.py
View File

@ -18,6 +18,7 @@ import subprocess
import yaml
from software.utilities import constants
from software.utilities.utils import configure_logging
import software.utilities.utils as utils
@ -44,10 +45,6 @@ DB_BARBICAN_CONNECTION_FORMAT = "postgresql://%s:%s@127.0.0.1:%s/%s"
# Configure logging
LOG = logging.getLogger(__name__)
log_format = ('%(asctime)s: ' + __name__ + '[%(process)s]: '
'%(filename)s(%(lineno)s): %(levelname)s: %(message)s')
log_datefmt = "%FT%T"
logging.basicConfig(filename="/var/log/software.log", format=log_format, level=logging.INFO, datefmt=log_datefmt)
def migrate_keyring_data(from_release, to_release):
@ -792,7 +789,7 @@ def upgrade_controller(from_release, to_release, target_port):
def migrate():
# this is the entry point to start data migration
configure_logging()
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("from_release",

25
software/software/utilities/utils.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2023 Wind River Systems, Inc.
# Copyright (c) 2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -29,25 +29,36 @@ KUBERNETES_ADMIN_CONF_FILE = "admin.conf"
PLATFORM_LOG = '/var/log/platform.log'
ERROR_FILE = '/tmp/upgrade_fail_msg'
SOFTWARE_LOG_FILE = "/var/log/software.log"
# well-known default domain name
DEFAULT_DOMAIN_NAME = 'Default'
# Migration script actions
# Upgrade script actions
ACTION_START = "start"
ACTION_MIGRATE = "migrate"
ACTION_ACTIVATE = "activate"
ACTION_ACTIVATE_ROLLBACK = "activate-rollback"
def configure_logging():
log_format = ('%(asctime)s: ' + __name__ + '[%(process)s]: '
'%(filename)s(%(lineno)s): %(levelname)s: %(message)s')
log_datefmt = "%FT%T"
logging.basicConfig(filename=SOFTWARE_LOG_FILE, format=log_format, level=logging.INFO, datefmt=log_datefmt)
def execute_migration_scripts(from_release, to_release, action, port=None,
migration_script_dir="/etc/upgrade.d"):
"""Execute migration scripts with an action:
"""Execute upgrade scripts with an action:
start: Prepare for upgrade on release N side. Called during
"system upgrade-start".
migrate: Perform data migration on release N+1 side. Called while
system data migration is taking place.
activate: Activates the deployment. Called during "software deploy activate".
activate-rollback: Rolls back the activate deployment. Called during
"software deploy activate".
"""
LOG.info("Executing migration scripts with from_release: %s, "
LOG.info("Executing upgrade scripts with from_release: %s, "
"to_release: %s, action: %s" % (from_release, to_release, action))
# Get a sorted list of all the migration scripts
@ -62,17 +73,17 @@ def execute_migration_scripts(from_release, to_release, action, port=None,
try:
files.sort(key=lambda x: int(x.split("-")[0]))
except Exception:
LOG.exception("Migration script sequence validation failed, invalid "
LOG.exception("Upgrade script sequence validation failed, invalid "
"file name format")
raise
MSG_SCRIPT_FAILURE = "Migration script %s failed with returncode %d" \
MSG_SCRIPT_FAILURE = "Upgrade script %s failed with returncode %d" \
"Script output:\n%s"
# Execute each migration script
for f in files:
migration_script = os.path.join(migration_script_dir, f)
try:
LOG.info("Executing migration script %s" % migration_script)
LOG.info("Executing upgrade script %s" % migration_script)
cmdline = [migration_script, from_release, to_release, action]
if port is not None:
cmdline.append(port)