diff --git a/software-client/software_client/v1/deploy.py b/software-client/software_client/v1/deploy.py index 3eabace5..ce21c787 100644 --- a/software-client/software_client/v1/deploy.py +++ b/software-client/software_client/v1/deploy.py @@ -94,6 +94,15 @@ class DeployManager(base.Manager): return self._create(path, body={}) + def activate_rollback(self, args): + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + # Issue deploy_start request + path = "/v1/deploy/activate_rollback" + + return self._create(path, body={}) + def complete(self, args): # Ignore interrupts during this function signal.signal(signal.SIGINT, signal.SIG_IGN) diff --git a/software-client/software_client/v1/deploy_cmd.py b/software-client/software_client/v1/deploy_cmd.py index 6be89a64..dd62caa7 100644 --- a/software-client/software_client/v1/deploy_cmd.py +++ b/software-client/software_client/v1/deploy_cmd.py @@ -19,6 +19,7 @@ DEPLOY_COMMAND_MODULES = [ # - host # - abort # - activate +# - activate-rollback # - complete # - delete # non root/sudo users can run: diff --git a/software-client/software_client/v1/deploy_shell.py b/software-client/software_client/v1/deploy_shell.py index 8697180e..2d2d4732 100644 --- a/software-client/software_client/v1/deploy_shell.py +++ b/software-client/software_client/v1/deploy_shell.py @@ -175,6 +175,17 @@ def do_activate(cc, args): return utils.check_rc(resp, data) +def do_activate_rollback(cc, args): + """Rolls back the activate of software deployment""" + resp, data = cc.deploy.activate_rollback(args) + if args.debug: + utils.print_result_debug(resp, data) + + utils.display_info(resp) + + return utils.check_rc(resp, data) + + def do_complete(cc, args): """Complete the software deployment""" resp, data = cc.deploy.complete(args) diff --git a/software/setup.cfg b/software/setup.cfg index 7c696f7b..79a49a9d 100644 --- a/software/setup.cfg +++ b/software/setup.cfg @@ -38,6 +38,7 @@ console_scripts = software-migrate = software.utilities.migrate:migrate software-deploy-update = software.utilities.update_deploy_state:update_state software-deploy-activate = software.utilities.activate:activate + software-deploy-activate-rollback = software.utilities.activate_rollback:activate_rollback [wheel] diff --git a/software/software/api/controllers/v1/deploy.py b/software/software/api/controllers/v1/deploy.py index dea8802a..84416fda 100644 --- a/software/software/api/controllers/v1/deploy.py +++ b/software/software/api/controllers/v1/deploy.py @@ -22,6 +22,7 @@ class DeployController(RestController): _custom_actions = { 'abort': ['POST'], 'activate': ['POST'], + 'activate_rollback': ['POST'], 'precheck': ['POST'], 'start': ['POST'], 'complete': ['POST'], @@ -45,6 +46,14 @@ class DeployController(RestController): sc.software_sync() return result + @expose(method='POST', template='json') + def activate_rollback(self): + reload_release_data() + + result = sc.software_deploy_activate_rollback_api() + sc.software_sync() + return result + @expose(method='POST', template='json') def complete(self): reload_release_data() diff --git a/software/software/deploy_state.py b/software/software/deploy_state.py index 21f3510e..00faa32f 100644 --- a/software/software/deploy_state.py +++ b/software/software/deploy_state.py @@ -46,14 +46,16 @@ deploy_state_transition = { DEPLOY_STATES.ACTIVATE: [DEPLOY_STATES.ACTIVATE_DONE, DEPLOY_STATES.ACTIVATE_FAILED], DEPLOY_STATES.ACTIVATE_FAILED: [DEPLOY_STATES.ACTIVATE, # deploy activate is reentrant DEPLOY_STATES.ACTIVATE_ROLLBACK], - DEPLOY_STATES.ACTIVATE_DONE: [DEPLOY_STATES.COMPLETED, DEPLOY_STATES.ACTIVATE_ROLLBACK], + DEPLOY_STATES.ACTIVATE_DONE: [DEPLOY_STATES.COMPLETED, DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING], # deploy activate rollback - DEPLOY_STATES.ACTIVATE_ROLLBACK: [DEPLOY_STATES.HOST_ROLLBACK, DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED], - DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED: [DEPLOY_STATES.ACTIVATE_ROLLBACK], # deploy host rollback is reentrant + DEPLOY_STATES.ACTIVATE_ROLLBACK: [DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE, DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED], + DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE: [DEPLOY_STATES.HOST_ROLLBACK], + DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED: [DEPLOY_STATES.ACTIVATE_ROLLBACK], # deploy activate rollback is reentrant + DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING: [DEPLOY_STATES.ACTIVATE_ROLLBACK], # deploy complete - DEPLOY_STATES.COMPLETED: [DEPLOY_STATES.ACTIVATE_ROLLBACK, None] + DEPLOY_STATES.COMPLETED: [DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING, None] } @@ -181,7 +183,7 @@ class DeployState(object): # host rollback, if post-activate then go to activate rollback state = DeployState.get_deploy_state() if state in [DEPLOY_STATES.ACTIVATE_DONE, DEPLOY_STATES.ACTIVATE_FAILED, DEPLOY_STATES.COMPLETED]: - self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK) + self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING) else: self.transform(DEPLOY_STATES.HOST_ROLLBACK) @@ -212,6 +214,9 @@ class DeployState(object): def activate_rollback(self): self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK) + def activate_rollback_done(self): + self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE) + def activate_rollback_failed(self): self.transform(DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED) diff --git a/software/software/software_controller.py b/software/software/software_controller.py index cf490046..8a4ea714 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -746,7 +746,7 @@ class SWMessageDeployStateChanged(messages.PatchMessage): self.valid = True self.agent = None - valid_agents = ['deploy-start', 'deploy-activate'] + valid_agents = ['deploy-start', 'deploy-activate', 'deploy-activate-rollback'] if 'agent' in data: self.agent = data['agent'] else: @@ -762,7 +762,9 @@ class SWMessageDeployStateChanged(messages.PatchMessage): DEPLOY_STATES.START_DONE.value: DEPLOY_STATES.START_DONE, DEPLOY_STATES.START_FAILED.value: DEPLOY_STATES.START_FAILED, DEPLOY_STATES.ACTIVATE_FAILED.value: DEPLOY_STATES.ACTIVATE_FAILED, - DEPLOY_STATES.ACTIVATE_DONE.value: DEPLOY_STATES.ACTIVATE_DONE + DEPLOY_STATES.ACTIVATE_DONE.value: DEPLOY_STATES.ACTIVATE_DONE, + DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE.value: DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE, + DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED.value: DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED } if 'deploy-state' in data and data['deploy-state']: deploy_state = data['deploy-state'] @@ -2413,7 +2415,9 @@ class PatchController(PatchService): DEPLOY_STATES.START_DONE: deploy_state.start_done, DEPLOY_STATES.START_FAILED: deploy_state.start_failed, DEPLOY_STATES.ACTIVATE_DONE: deploy_state.activate_done, - DEPLOY_STATES.ACTIVATE_FAILED: deploy_state.activate_failed + DEPLOY_STATES.ACTIVATE_FAILED: deploy_state.activate_failed, + DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE: deploy_state.activate_rollback_done, + DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED: deploy_state.activate_rollback_failed } if new_state in state_event: state_event[new_state]() @@ -3013,6 +3017,61 @@ class PatchController(PatchService): return dict(info=msg_info, warning=msg_warning, error=msg_error) + def _activate_rollback_major_release(self, deploy): + cmd_path = "/usr/bin/software-deploy-activate-rollback" + from_release = utils.get_major_release_version(deploy.get("from_release")) + to_release = utils.get_major_release_version(deploy.get("to_release")) + + upgrade_activate_rollback_cmd = [cmd_path, from_release, to_release] + + try: + LOG.info("starting subprocess %s" % ' '.join(upgrade_activate_rollback_cmd)) + subprocess.Popen(' '.join(upgrade_activate_rollback_cmd), start_new_session=True, shell=True) + LOG.info("subprocess started") + except subprocess.SubprocessError as e: + LOG.error("Failed to start command: %s. Error %s" % (' '.join(upgrade_activate_rollback_cmd), e)) + raise + + def _activate_rollback_patching_release(self): + deploy_state = DeployState.get_instance() + # patching release activate-rollback operations go here + deploy_state.activate_rollback_done() + + def _activate_rollback(self): + deploy = self.db_api_instance.get_current_deploy() + if not deploy: + msg = "Deployment is missing unexpectedly" + raise InvalidOperation(msg) + + deploying = ReleaseState(release_state=states.DEPLOYING) + if deploying.is_major_release_deployment(): + self._activate_rollback_major_release(deploy) + else: + self._activate_rollback_patching_release() + + @require_deploy_state([DEPLOY_STATES.ACTIVATE_ROLLBACK_PENDING, DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED], + "Activate-rollback deployment only when current deployment state is {require_states}") + def software_deploy_activate_rollback_api(self) -> dict: + """ + Rolls back activates the deployment associated with the release + :return: dict of info, warning and error messages + """ + msg_info = "" + msg_warning = "" + msg_error = "" + + deploy_state = DeployState.get_instance() + deploy_state.activate_rollback() + + try: + self._activate_rollback() + msg_info = "Deploy activate-rollback has started" + except Exception: + deploy_state.activate_rollback_failed() + raise + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + def software_deploy_show_api(self, from_release=None, to_release=None): # Retrieve deploy state from db if from_release and to_release: diff --git a/software/software/states.py b/software/software/states.py index 9e09a83a..fc85c320 100644 --- a/software/software/states.py +++ b/software/software/states.py @@ -119,7 +119,9 @@ class DEPLOY_STATES(Enum): ACTIVATE_FAILED = 'activate-failed' ACTIVATE_ROLLBACK = 'activate-rollback' + ACTIVATE_ROLLBACK_DONE = 'activate-rollback-done' ACTIVATE_ROLLBACK_FAILED = 'activate-rollback-failed' + ACTIVATE_ROLLBACK_PENDING = 'activate-rollback-pending' COMPLETED = 'completed' diff --git a/software/software/utilities/activate.py b/software/software/utilities/activate.py index 3057b03f..23e0b5f4 100644 --- a/software/software/utilities/activate.py +++ b/software/software/utilities/activate.py @@ -9,6 +9,7 @@ from oslo_log import log from software.states import DEPLOY_STATES from software.utilities.update_deploy_state import update_deploy_state +from software.utilities.utils import configure_logging from software.utilities.utils import execute_migration_scripts from software.utilities.utils import ACTION_ACTIVATE @@ -38,8 +39,8 @@ def do_activate(from_release, to_release): def activate(): - # this is the entry point to start data migration - + # this is the entry point to start activate + configure_logging() parser = argparse.ArgumentParser(add_help=False) parser.add_argument("from_release", diff --git a/software/software/utilities/activate_rollback.py b/software/software/utilities/activate_rollback.py new file mode 100644 index 00000000..5626cac3 --- /dev/null +++ b/software/software/utilities/activate_rollback.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import argparse + +from oslo_log import log + +from software.states import DEPLOY_STATES +from software.utilities.update_deploy_state import update_deploy_state +from software.utilities.utils import configure_logging +from software.utilities.utils import execute_migration_scripts +from software.utilities.utils import ACTION_ACTIVATE_ROLLBACK + +LOG = log.getLogger(__name__) + + +def do_activate_rollback(from_release, to_release): + agent = 'deploy-activate-rollback' + res = True + state = DEPLOY_STATES.ACTIVATE_ROLLBACK_DONE.value + try: + execute_migration_scripts(from_release, to_release, ACTION_ACTIVATE_ROLLBACK) + except Exception: + state = DEPLOY_STATES.ACTIVATE_ROLLBACK_FAILED.value + res = False + finally: + try: + update_deploy_state(agent, deploy_state=state) + if res: + LOG.info("Deploy activate-rollback completed successfully") + else: + LOG.error("Deploy activate-rollback failed") + except Exception as err: + LOG.error("Update deploy state activate-rollback failed: %s" % err) + res = False + return res + + +def activate_rollback(): + # this is the entry point to start activate-rollback + configure_logging() + parser = argparse.ArgumentParser(add_help=False) + + parser.add_argument("from_release", + default=False, + help="From release") + + parser.add_argument("to_release", + default=False, + help="To release") + + args = parser.parse_args() + + if do_activate_rollback(args.from_release, args.to_release): + exit(0) + else: + exit(1) diff --git a/software/software/utilities/migrate.py b/software/software/utilities/migrate.py index 38cd0f1d..1f07dd0d 100644 --- a/software/software/utilities/migrate.py +++ b/software/software/utilities/migrate.py @@ -18,6 +18,7 @@ import subprocess import yaml from software.utilities import constants +from software.utilities.utils import configure_logging import software.utilities.utils as utils @@ -44,10 +45,6 @@ DB_BARBICAN_CONNECTION_FORMAT = "postgresql://%s:%s@127.0.0.1:%s/%s" # Configure logging LOG = logging.getLogger(__name__) -log_format = ('%(asctime)s: ' + __name__ + '[%(process)s]: ' - '%(filename)s(%(lineno)s): %(levelname)s: %(message)s') -log_datefmt = "%FT%T" -logging.basicConfig(filename="/var/log/software.log", format=log_format, level=logging.INFO, datefmt=log_datefmt) def migrate_keyring_data(from_release, to_release): @@ -792,7 +789,7 @@ def upgrade_controller(from_release, to_release, target_port): def migrate(): # this is the entry point to start data migration - + configure_logging() parser = argparse.ArgumentParser(add_help=False) parser.add_argument("from_release", diff --git a/software/software/utilities/utils.py b/software/software/utilities/utils.py index 433d652c..d1c610e6 100644 --- a/software/software/utilities/utils.py +++ b/software/software/utilities/utils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Wind River Systems, Inc. +# Copyright (c) 2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -29,25 +29,36 @@ KUBERNETES_ADMIN_CONF_FILE = "admin.conf" PLATFORM_LOG = '/var/log/platform.log' ERROR_FILE = '/tmp/upgrade_fail_msg' +SOFTWARE_LOG_FILE = "/var/log/software.log" + # well-known default domain name DEFAULT_DOMAIN_NAME = 'Default' -# Migration script actions +# Upgrade script actions ACTION_START = "start" ACTION_MIGRATE = "migrate" ACTION_ACTIVATE = "activate" +ACTION_ACTIVATE_ROLLBACK = "activate-rollback" +def configure_logging(): + log_format = ('%(asctime)s: ' + __name__ + '[%(process)s]: ' + '%(filename)s(%(lineno)s): %(levelname)s: %(message)s') + log_datefmt = "%FT%T" + logging.basicConfig(filename=SOFTWARE_LOG_FILE, format=log_format, level=logging.INFO, datefmt=log_datefmt) def execute_migration_scripts(from_release, to_release, action, port=None, migration_script_dir="/etc/upgrade.d"): - """Execute migration scripts with an action: + """Execute upgrade scripts with an action: start: Prepare for upgrade on release N side. Called during "system upgrade-start". migrate: Perform data migration on release N+1 side. Called while system data migration is taking place. + activate: Activates the deployment. Called during "software deploy activate". + activate-rollback: Rolls back the activate deployment. Called during + "software deploy activate". """ - LOG.info("Executing migration scripts with from_release: %s, " + LOG.info("Executing upgrade scripts with from_release: %s, " "to_release: %s, action: %s" % (from_release, to_release, action)) # Get a sorted list of all the migration scripts @@ -62,17 +73,17 @@ def execute_migration_scripts(from_release, to_release, action, port=None, try: files.sort(key=lambda x: int(x.split("-")[0])) except Exception: - LOG.exception("Migration script sequence validation failed, invalid " + LOG.exception("Upgrade script sequence validation failed, invalid " "file name format") raise - MSG_SCRIPT_FAILURE = "Migration script %s failed with returncode %d" \ + MSG_SCRIPT_FAILURE = "Upgrade script %s failed with returncode %d" \ "Script output:\n%s" # Execute each migration script for f in files: migration_script = os.path.join(migration_script_dir, f) try: - LOG.info("Executing migration script %s" % migration_script) + LOG.info("Executing upgrade script %s" % migration_script) cmdline = [migration_script, from_release, to_release, action] if port is not None: cmdline.append(port)