starlingx/update
Code Issues Proposed changes

data migration worker

Browse Source 
This commit is a worker process to perform data migration.
This process happens on the active controller running from-release s/w,
in a chroot environment with to-release s/w.
This subprocess takes from-release, to-release and port of 2nd instance
of PostgreSQL service (which runs to-release s/w and environment) as
command line.
The worker process creates PostgreSQL databases, tables and import data
from existing system, migrate keyring, pxeboot, helm, armada, sysinv,
fluxcd and hieradata.

TCs
    passed: successfully complete data migration, verify above
mentioned data are created correctly.

Depends-on: https://review.opendev.org/c/starlingx/config/+/896959
Depends-on: https://review.opendev.org/c/starlingx/config/+/897580
Depends-on: https://review.opendev.org/c/starlingx/update/+/883978
Depends-on: https://review.opendev.org/c/starlingx/config/+/899076
Story: 2010676
Task: 48837

Change-Id: I8c9205df27e68f7185ec154ee365101e33610a69
This commit is contained in:
Bin Qian 2023-09-26 20:02:28 +00:00
parent 40ca47d03d
commit f4f7daff48
7 changed files with 1247 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
software/debian/deb_folder/rules
View File

@ -14,6 +14,7 @@ export METADATA_FILE="starlingx-${PLATFORM_RELEASE}.0-metadata.xml"
override_dh_auto_build:
cp service-files/starlingx-GA-metadata.xml ${METADATA_FILE}
sed -i "s/xxxPLATFORM_RELEASExxx/${PLATFORM_RELEASE}/g" ${METADATA_FILE}
sed -i "s/xxxPLATFORM_RELEASExxx/${PLATFORM_RELEASE}/g" software/utilities/constants.py
override_dh_install:
python3 setup.py install -f --install-layout=deb --root=$(ROOT)

2
software/requirements.txt
View File

@ -1,4 +1,5 @@
argcomplete
keyring
keystoneauth1
keystonemiddleware
lxml
@ -7,6 +8,7 @@ oslo.policy
oslo.serialization
netaddr
pecan
psycopg2-binary
pycryptodomex
PyGObject
requests_toolbelt

1
software/setup.cfg
View File

@ -35,6 +35,7 @@ exclude =
console_scripts =
software-controller-daemon = software.software_controller:main
software-agent = software.software_agent:main
software-migrate = software.utilities.migrate:migrate
[wheel]

0
software/software/utilities/__init__.py Normal file
View File

37
software/software/utilities/constants.py Normal file
View File

@ -0,0 +1,37 @@
#
# Copyright (c) 2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# SW_VERSION should be built-in from build-tools
SW_VERSION = 'xxxPLATFORM_RELEASExxx'
DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER = 'systemcontroller'
WORKER = 'worker'
SERVICE_TYPE_IDENTITY = 'identity'
PLATFORM_PATH = "/opt/platform"
PLATFORM_CONFIG_PATH = PLATFORM_PATH + "/config/" + SW_VERSION + "/"
PLATFORM_CONF_FILE = PLATFORM_CONFIG_PATH + "/platform.conf"
CONFIG_PATH = PLATFORM_PATH + "/config/" + SW_VERSION + "/"
PLATFORM_CONFIG_PERMDIR = CONFIG_PATH
PUPPET_PATH = PLATFORM_PATH + "/puppet/" + SW_VERSION + "/"
HIERADATA_PERMDIR = PUPPET_PATH + 'hieradata'
KEYRING_WORKDIR = '/tmp/python_keyring'
KEYRING_PATH = PLATFORM_PATH + "/.keyring/" + SW_VERSION
KEYRING_PERMDIR = KEYRING_PATH
VOLATILE_PXEBOOT_PATH = "/var/pxeboot"
KUBERNETES_CONF_PATH = "/etc/kubernetes"
KUBERNETES_ADMIN_CONF_FILE = "admin.conf"
CONTROLLER = 'controller'
CONTROLLER_0_HOSTNAME = 'controller-0'
CONTROLLER_1_HOSTNAME = 'controller-1'
SIMPLEX = 'simplex'
DUPLEX = 'duplex'

792
software/software/utilities/migrate.py Normal file
View File

@ -0,0 +1,792 @@
#
# Copyright (c) 2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import argparse
import glob
import json
import os
from pathlib import Path
import psycopg2
import shutil
import socket
import sys
import subprocess
import yaml
from oslo_log import log
from software.utilities import constants
import software.utilities.utils as utils
sout = sys.stdout
def get_postgres_bin():
"""Get the path to the postgres binaries"""
try:
return subprocess.check_output(
['pg_config', '--bindir']).decode().rstrip('\n')
except subprocess.CalledProcessError:
LOG.exception("Failed to get postgres bin directory.")
raise
POSTGRES_BIN = get_postgres_bin()
LOG = log.getLogger(__name__)
POSTGRES_PATH = '/var/lib/postgresql'
POSTGRES_DATA_DIR = os.path.join(POSTGRES_PATH, constants.SW_VERSION)
DB_CONNECTION_FORMAT = "connection=postgresql://%s:%s@127.0.0.1:%s/%s\n"
DB_BARBICAN_CONNECTION_FORMAT = "postgresql://%s:%s@127.0.0.1:%s/%s"
def migrate_keyring_data(from_release, to_release):
"""Migrates keyring data. """
LOG.info("Migrating keyring data")
# First delete any keyring files for the to_release - they can be created
# if release N+1 nodes are incorrectly left powered up when the release N
# load is installed.
target_path = os.path.join(constants.PLATFORM_PATH, ".keyring", to_release)
shutil.rmtree(target_path, ignore_errors=True)
shutil.copytree(os.path.join(constants.PLATFORM_PATH, ".keyring", from_release), target_path)
def migrate_pxeboot_config(from_release, to_release):
"""Migrates pxeboot configuration. """
LOG.info("Migrating pxeboot config")
# Copy the entire pxelinux.cfg directory to pick up any changes made
# after the data was migrated (i.e. updates to the controller-1 load).
source_pxelinux = os.path.join(constants.PLATFORM_PATH, "config", from_release,
"pxelinux.cfg", "")
dest_pxelinux = os.path.join(constants.PLATFORM_PATH, "config", to_release,
"pxelinux.cfg")
Path(dest_pxelinux).mkdir(parents=True, exist_ok=True)
try:
subprocess.check_call(
["rsync",
"-a",
os.path.join(source_pxelinux),
os.path.join(dest_pxelinux)],
stdout=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to migrate %s" % source_pxelinux)
raise
to_release_symlink_target = os.path.join(constants.VOLATILE_PXEBOOT_PATH,
"pxelinux.cfg.files", "grub.cfg")
dest_symlink_exists = os.path.islink(dest_pxelinux + "/grub.cfg")
if dest_symlink_exists:
os.unlink(dest_pxelinux + "/grub.cfg")
os.symlink(to_release_symlink_target, dest_pxelinux + "/grub.cfg")
def migrate_armada_config(from_release, to_release):
"""Migrates armada configuration. """
# Check if the folder exist before migration
if not os.path.exists(os.path.join(constants.PLATFORM_PATH, "armada")):
LOG.info("Skipping armada migration, the directory doesn't exist")
return
LOG.info("Migrating armada config")
# Copy the entire armada.cfg directory to pick up any changes made
# after the data was migrated (i.e. updates to the controller-1 load).
source_armada = os.path.join(constants.PLATFORM_PATH, "armada", from_release, "")
dest_armada = os.path.join(constants.PLATFORM_PATH, "armada", to_release)
try:
subprocess.check_call(
["rsync",
"-a",
os.path.join(source_armada),
os.path.join(dest_armada)],
stdout=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to migrate %s" % source_armada)
raise
def migrate_fluxcd_config(from_release, to_release):
"""Migrates fluxcd configuration. """
# Check if the folder exists before migration
if not os.path.exists(os.path.join(constants.PLATFORM_PATH, "fluxcd")):
LOG.info("Skipping fluxcd migration, the directory doesn't exist")
return
LOG.info("Migrating fluxcd config")
# Copy the entire fluxcd.cfg directory to pick up any changes made
# after the data was migrated.
source_fluxcd = os.path.join(constants.PLATFORM_PATH, "fluxcd", from_release, "")
dest_fluxcd = os.path.join(constants.PLATFORM_PATH, "fluxcd", to_release)
try:
subprocess.check_call(
["rsync",
"-a",
os.path.join(source_fluxcd),
os.path.join(dest_fluxcd)],
stdout=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to migrate %s" % source_fluxcd)
raise
def migrate_helm_config(from_release, to_release):
"""Migrates helm configuration. """
LOG.info("Migrating helm config")
# Copy the entire helm.cfg directory to pick up any changes made
# after the data was migrated (i.e. updates to the controller-1 load).
source_helm = os.path.join(constants.PLATFORM_PATH, "helm", from_release, "")
dest_helm = os.path.join(constants.PLATFORM_PATH, "helm", to_release)
try:
subprocess.check_call(
["rsync",
"-a",
os.path.join(source_helm),
os.path.join(dest_helm)],
stdout=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to migrate %s" % source_helm)
raise
def migrate_sysinv_data(from_release, to_release, port):
"""Migrates sysinv data. """
LOG.info("Migrating sysinv data")
# If the /opt/platform/sysinv/<release>/sysinv.conf.default file has
# changed between releases it must be modified at this point.
try:
subprocess.check_call(
["rsync",
"-a",
os.path.join(constants.PLATFORM_PATH, "sysinv", from_release, ""),
os.path.join(constants.PLATFORM_PATH, "sysinv", to_release)],
stdout=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to copy sysinv platform dir to new version")
raise
# Get the hiera data for the from release
hiera_path = os.path.join(constants.PLATFORM_PATH, "puppet", from_release,
"hieradata")
static_file = os.path.join(hiera_path, "static.yaml")
with open(static_file, 'r') as file:
static_config = yaml.load(file, Loader=yaml.Loader)
username = static_config["sysinv::db::postgresql::user"]
password = utils.get_password_from_keyring("sysinv", "database")
# We need a bare bones /etc/sysinv/sysinv.conf file in order to do the
# sysinv database migration and then generate the upgrades manifests.
with open("/etc/sysinv/sysinv.conf", "w") as f:
f.write("[DEFAULT]\n")
f.write("logging_context_format_string=sysinv %(asctime)s.%"
"(msecs)03d %(process)d %(levelname)s %"
"(name)s [%(request_id)s %(user)s %"
"(tenant)s] %(instance)s%(message)s\n")
f.write("verbose=True\n")
f.write("syslog_log_facility=local6\n")
f.write("use_syslog=True\n")
f.write("logging_default_format_string=sysinv %(asctime)s.%"
"(msecs)03d %(process)d %(levelname)s %(name)s [-] %"
"(instance)s%(message)s\n")
f.write("debug=False\n")
f.write('sql_connection=postgresql://%s:%s@127.0.0.1:%s/sysinv\n' %
(username, password, port))
def create_database(target_port):
"""Creates empty postgres database. """
LOG.info("Creating postgres database")
db_create_commands = [
# Configure new data directory for postgres
'rm -rf {}'.format(POSTGRES_DATA_DIR),
'mkdir -p {}'.format(POSTGRES_DATA_DIR),
'chown postgres {}'.format(POSTGRES_DATA_DIR),
'sudo -u postgres {} -D {}'.format(
os.path.join(POSTGRES_BIN, 'initdb'),
POSTGRES_DATA_DIR),
'chmod -R 700 ' + POSTGRES_DATA_DIR,
'chown -R postgres ' + POSTGRES_DATA_DIR,
"sed -i 's/#port = 5432/port = {}/g' {}/postgresql.conf".format(target_port, POSTGRES_DATA_DIR),
'mkdir -p /var/run/postgresql/',
'chown -R postgres /var/run/postgresql',
]
# Execute db creation commands
for cmd in db_create_commands:
try:
LOG.info("Executing db create command: %s" % cmd)
subprocess.check_call([cmd],
shell=True, stdout=sout, stderr=sout)
except subprocess.CalledProcessError as ex:
LOG.exception("Failed to execute command: '%s' during upgrade "
"processing, return code: %d" % (cmd, ex.returncode))
raise
def import_databases(target_port, from_path=None):
"""Imports databases. """
if not from_path:
from_dir = '/var/lib/postgresql/upgrade/'
LOG.info("Importing databases")
try:
postgres_config_path = os.path.join(
from_dir, 'postgres.postgreSql.config')
# Do postgres schema import (suppress stderr due to noise)
subprocess.check_call(['sudo -u postgres psql --port=%s -f ' % target_port +
postgres_config_path + ' postgres'],
shell=True,
stdout=sout,
stderr=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to import schemas.")
raise
import_commands = []
# Do postgres data import
for data in glob.glob(from_dir + '/*.*Sql.data'):
db_elem = data.split('/')[-1].split('.')[0]
LOG.info("importing {}".format(db_elem))
import_commands.append((db_elem,
"sudo -u postgres psql --port=%s -f " % target_port + data +
" " + db_elem))
VIM_DB_NAME = 'vim_db_v1'
temp_db_path = '/tmp/'
db_dir = os.path.join(constants.PLATFORM_PATH, 'nfv/vim', constants.SW_VERSION)
db_path = os.path.join(db_dir, VIM_DB_NAME)
import_commands.append(
("remove %s" % db_dir,
"rm %s -rf" % db_dir))
import_commands.append(
("create %s" % db_dir,
"mkdir %s -p" % db_dir))
import_commands.append(
("nfv-vim",
"nfv-vim-manage db-load-data -d %s -f %s" %
(temp_db_path, os.path.join(from_dir, 'vim.data'))))
# copy the vim db
import_commands.append(
('move database to %s' % db_path,
("mv %s %s" % (os.path.join(temp_db_path, VIM_DB_NAME),
db_path))))
# Execute import commands
for cmd in import_commands:
try:
print("Importing %s" % cmd[0])
LOG.info("Executing import command: %s" % cmd[1])
subprocess.check_call([cmd[1]],
shell=True, stdout=sout)
except subprocess.CalledProcessError as ex:
LOG.exception("Failed to execute command: '%s' during upgrade "
"processing, return code: %d" %
(cmd[1], ex.returncode))
raise
def get_system_role(target_port):
"""Get the system role from the sysinv database"""
conn = psycopg2.connect("dbname=sysinv user=postgres port=%s" % target_port)
cur = conn.cursor()
cur.execute("select distributed_cloud_role from i_system;")
row = cur.fetchone()
if row is None:
LOG.error("Failed to fetch i_system data")
raise psycopg2.ProgrammingError("Failed to fetch i_system data")
role = row[0]
return role
def get_system_mode(target_port):
"""Get the system mode (simplex or duplex)
from the sysinv database
"""
conn = psycopg2.connect("dbname=sysinv user=postgres port=%s" % target_port)
cur = conn.cursor()
cur.execute("select system_mode from i_system;")
row = cur.fetchone()
if row is None:
LOG.error("Failed to fetch i_system data")
raise psycopg2.ProgrammingError("Failed to fetch i_system data")
role = row[0]
return role
def get_first_controller(target_port):
"""retrieve the first controller to upgrade.
in sx, controller-0, or
in dx, controller-1
"""
system_mode = get_system_mode(target_port)
if system_mode == constants.SIMPLEX:
return constants.CONTROLLER_0_HOSTNAME
else:
return constants.CONTROLLER_1_HOSTNAME
def get_shared_services(target_port):
"""Get the list of shared services from the sysinv database"""
shared_services = []
DEFAULT_SHARED_SERVICES = []
conn = psycopg2.connect("dbname=sysinv user=postgres port=%s" % target_port)
cur = conn.cursor()
cur.execute("select capabilities from i_system;")
row = cur.fetchone()
if row is None:
LOG.error("Failed to fetch i_system data")
raise psycopg2.ProgrammingError("Failed to fetch i_system data")
cap_obj = json.loads(row[0])
region_config = cap_obj.get('region_config', None)
if region_config:
shared_services = cap_obj.get('shared_services',
DEFAULT_SHARED_SERVICES)
return shared_services
def migrate_hiera_data(from_release):
"""Migrate static hiera data. """
LOG.info("Migrating hiera data")
from_hiera_path = os.path.join(constants.PLATFORM_PATH, "puppet", from_release,
"hieradata")
to_hiera_path = constants.HIERADATA_PERMDIR
shutil.rmtree(to_hiera_path, ignore_errors=True)
os.makedirs(to_hiera_path)
# Copy only the static yaml files. The other yaml files will be generated
# when required.
for f in ['secure_static.yaml', 'static.yaml']:
shutil.copy(os.path.join(from_hiera_path, f), to_hiera_path)
# Make any necessary updates to the static yaml files.
# Update the static.yaml file
static_file = os.path.join(constants.HIERADATA_PERMDIR, "static.yaml")
with open(static_file, 'r') as yaml_file:
static_config = yaml.load(yaml_file, Loader=yaml.Loader)
static_config.update({
'platform::params::software_version': constants.SW_VERSION,
'platform::client::credentials::params::keyring_directory':
constants.KEYRING_PATH,
'platform::client::credentials::params::keyring_file':
os.path.join(constants.KEYRING_PATH, '.CREDENTIAL'),
})
with open(static_file, 'w') as yaml_file:
yaml.dump(static_config, yaml_file, default_flow_style=False)
secure_static_file = os.path.join(
constants.HIERADATA_PERMDIR, "secure_static.yaml")
with open(secure_static_file, 'r') as yaml_file:
secure_static_config = yaml.load(yaml_file, Loader=yaml.Loader)
with open(secure_static_file, 'w') as yaml_file:
yaml.dump(secure_static_config, yaml_file, default_flow_style=False)
def get_db_credentials(shared_services, from_release, role=None):
"""
Returns the database credentials using the provided shared services,
from_release and role.
"""
db_credential_keys = \
{'barbican': {'hiera_user_key': 'barbican::db::postgresql::user',
'keyring_password_key': 'barbican',
},
'sysinv': {'hiera_user_key': 'sysinv::db::postgresql::user',
'keyring_password_key': 'sysinv',
},
'fm': {'hiera_user_key': 'fm::db::postgresql::user',
'keyring_password_key': 'fm',
},
}
if constants.SERVICE_TYPE_IDENTITY not in shared_services:
db_credential_keys.update(
{'keystone': {'hiera_user_key':
'keystone::db::postgresql::user',
'keyring_password_key': 'keystone',
}})
if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
db_credential_keys.update(
{'dcmanager': {'hiera_user_key': 'dcmanager::db::postgresql::user',
'keyring_password_key': 'dcmanager',
},
'dcorch': {'hiera_user_key': 'dcorch::db::postgresql::user',
'keyring_password_key': 'dcorch',
},
})
# Get the hiera data for the from release
hiera_path = os.path.join(constants.PLATFORM_PATH, "puppet", from_release,
"hieradata")
static_file = os.path.join(hiera_path, "static.yaml")
with open(static_file, 'r') as file:
static_config = yaml.load(file, Loader=yaml.Loader)
db_credentials = dict()
for database, values in db_credential_keys.items():
username = static_config[values['hiera_user_key']]
password = utils.get_password_from_keyring(
values['keyring_password_key'], "database")
db_credentials[database] = {'username': username, 'password': password}
return db_credentials
def create_databases(db_credentials):
"""Creates databases. """
LOG.info("Creating new databases")
# Create databases that are new in this release
conn = psycopg2.connect('dbname=postgres user=postgres port=6666')
# Postgres won't allow transactions around database create operations
# so we set the connection to autocommit
conn.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
databases_to_create = []
if not databases_to_create:
return
with conn:
with conn.cursor() as cur:
for database in databases_to_create:
print("Creating %s database" % database)
username = psycopg2.extensions.AsIs(
'\"%s\"' % db_credentials[database]['username'])
db_name = psycopg2.extensions.AsIs('\"%s\"' % database)
password = db_credentials[database]['password']
try:
# Here we create the new database and the role for it
# The role will be used by the dbsync command to
# connect to the database. This ensures any new tables
# are added with the correct owner
cur.execute('CREATE DATABASE %s', (db_name,))
cur.execute('CREATE ROLE %s', (username,))
cur.execute('ALTER ROLE %s LOGIN PASSWORD %s',
(username, password))
cur.execute('GRANT ALL ON DATABASE %s TO %s',
(db_name, username))
except Exception as ex:
LOG.exception("Failed to create database and role. " +
"(%s : %s) Exception: %s" %
(database, username, ex))
raise
def migrate_sysinv_database():
"""Migrates the sysinv database. """
sysinv_cmd = 'sysinv-dbsync'
try:
print("Migrating sysinv")
LOG.info("Executing migrate command: %s" % sysinv_cmd)
subprocess.check_call(sysinv_cmd,
shell=True, stdout=sout, stderr=sout)
except subprocess.CalledProcessError as ex:
LOG.exception("Failed to execute command: '%s' during upgrade "
"processing, return code: %d"
% (sysinv_cmd, ex.returncode))
raise
def migrate_databases(shared_services, db_credentials, port,
simplex=False, role=None):
"""Migrates databases. """
# Create minimal config files for each OpenStack service so they can
# run their database migration.
if constants.SERVICE_TYPE_IDENTITY not in shared_services:
with open("/etc/keystone/keystone-dbsync.conf", "w") as f:
f.write("[database]\n")
f.write(get_connection_string(db_credentials, port, 'keystone'))
migrate_commands = [
# Migrate barbican
('barbican',
'barbican-manage db upgrade ' +
'--db-url %s' % get_connection_string(db_credentials, port, 'barbican')),
]
# Migrate fm
# append the migrate command for dcmanager db
with open("/etc/fm/fm.conf", "w") as f:
f.write("[database]\n")
f.write(get_connection_string(db_credentials, port, 'fm'))
migrate_commands += [
('fm',
'fm-dbsync')
]
if constants.SERVICE_TYPE_IDENTITY not in shared_services:
# To avoid a deadlock during keystone contract we will use offline
# migration for simplex upgrades. Other upgrades will have to use
# another method to resolve the deadlock
if not simplex:
migrate_commands += [
# Migrate keystone
#
# EXPAND - we will first expand the database scheme to a
# superset of what both the previous and next release can
# utilize, and create triggers to facilitate the live
# migration process.
#
# MIGRATE - will perform the data migration, while still]
# preserving the old schema
('keystone',
'keystone-manage --config-file ' +
'/etc/keystone/keystone-dbsync.conf db_sync --expand'),
('keystone',
'keystone-manage --config-file ' +
'/etc/keystone/keystone-dbsync.conf db_sync --migrate'),
]
else:
migrate_commands += [
# In simplex we're the only node so we can do an offline
# migration
('keystone',
'keystone-manage --config-file ' +
'/etc/keystone/keystone-dbsync.conf db_sync')
]
if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
# append the migrate command for dcmanager db
with open("/etc/dcmanager/dcmanager.conf", "w") as f:
f.write("[database]\n")
f.write(get_connection_string(db_credentials, port, 'dcmanager'))
migrate_commands += [
('dcmanager',
'dcmanager-manage db_sync')
]
# append the migrate command for dcorch db
with open("/etc/dcorch/dcorch.conf", "w") as f:
f.write("[database]\n")
f.write(get_connection_string(db_credentials, port, 'dcorch'))
migrate_commands += [
('dcorch',
'dcorch-manage db_sync')
]
# Execute migrate commands
for cmd in migrate_commands:
try:
print("Migrating %s" % cmd[0])
LOG.info("Executing migrate command: %s" % cmd[1])
subprocess.check_call([cmd[1]],
shell=True, stdout=sout, stderr=sout)
except subprocess.CalledProcessError as ex:
LOG.exception("Failed to execute command: '%s' during upgrade "
"processing, return code: %d" %
(cmd[1], ex.returncode))
raise
# The database entry for the first controller to be upgrade will be set
# to whatever it was when the sysinv database was dumped on. Update the
# state and from/to load to what it should be when it becomes active.
try:
subprocess.check_call(
["/usr/bin/sysinv-upgrade",
"update_controller_state",
"--skip_load_update"],
stdout=sout, stderr=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to update state of the controller")
raise
def gethostaddress(hostname):
"""Get the IP address for a hostname, supporting IPv4 and IPv6. """
return socket.getaddrinfo(hostname, None)[0][4][0]
def get_connection_string(db_credentials, port, database):
"""Generates a connection string for a given database"""
username = db_credentials[database]['username']
password = db_credentials[database]['password']
if database == 'barbican':
return DB_BARBICAN_CONNECTION_FORMAT % (username, password, port, database)
else:
return DB_CONNECTION_FORMAT % (username, password, port, database)
def upgrade_controller(from_release, to_release, target_port):
"""Executed on controller-0, under chroot N+1 deployment and N runtime. """
LOG.info("Upgrading controller from %s to %s" % (from_release, to_release))
LOG.info("Mounting filesystems already done before chroot")
# Migrate keyring data
print("Migrating keyring data...")
migrate_keyring_data(from_release, to_release)
# Migrate pxeboot config
print("Migrating pxeboot configuration...")
migrate_pxeboot_config(from_release, to_release)
# Migrate armada config
print("Migrating armada configuration...")
migrate_armada_config(from_release, to_release)
# Migrate fluxcd config
print("Migrating fluxcd configuration...")
migrate_fluxcd_config(from_release, to_release)
# Migrate helm config
print("Migrating helm configuration...")
migrate_helm_config(from_release, to_release)
# Migrate sysinv data.
print("Migrating sysinv configuration...")
migrate_sysinv_data(from_release, to_release, target_port)
# Prepare for database migration
print("Preparing for database migration...")
# prepare_postgres_filesystems()
# Import databases
print("Importing databases...")
import_databases(target_port)
role = get_system_role(target_port)
shared_services = get_shared_services(target_port)
# Create /tmp/python_keyring - used by keystone manifest.
tmp_keyring_path = "/tmp/python_keyring"
key_ring_path = os.path.join(constants.PLATFORM_PATH, ".keyring", to_release,
"python_keyring")
shutil.rmtree(tmp_keyring_path, ignore_errors=True)
shutil.copytree(key_ring_path, tmp_keyring_path)
# Migrate hiera data
migrate_hiera_data(from_release)
utils.add_upgrade_entries_to_hiera_data(from_release)
# Get database credentials
db_credentials = get_db_credentials(
shared_services, from_release, role=role)
# Create any new databases
print("Creating new databases...")
create_databases(db_credentials)
print("Migrating databases...")
# Migrate sysinv database
migrate_sysinv_database()
# Migrate databases
migrate_databases(shared_services, db_credentials, target_port, role=role)
print("Applying configuration...")
# Execute migration scripts
utils.execute_migration_scripts(
from_release, to_release, utils.ACTION_MIGRATE, target_port)
# Remove manifest and keyring files
shutil.rmtree("/tmp/python_keyring", ignore_errors=True)
first_controller = get_first_controller(target_port)
# Generate config to be used by "regular" manifest
print("Generating config for %s" % first_controller)
LOG.info("Generating config for %s" % first_controller)
try:
utils.create_system_config()
utils.create_host_config(first_controller)
except Exception as e:
LOG.exception(e)
LOG.info("Failed to update hiera configuration")
raise
# Stop postgres server
LOG.info("Shutting down PostgreSQL...")
try:
subprocess.check_call([
'sudo',
'-u',
'postgres',
os.path.join(POSTGRES_BIN, 'pg_ctl'),
'-D',
POSTGRES_DATA_DIR,
'stop'],
stdout=sout)
except subprocess.CalledProcessError:
LOG.exception("Failed to stop postgres service")
raise
print("Data migration complete !!!")
LOG.info("Data migration complete !!!")
def migrate():
# this is the entry point to start data migration
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("from_release",
default=False,
help="From release")
parser.add_argument("to_release",
default=False,
help="To release")
parser.add_argument('port',
default=6666,
help="PostgreSQL service port to access target database.")
parser.add_argument('-v', '--verbose',
default=False, action="store_true",
help="Print more verbose output")
args = parser.parse_args()
upgrade_controller(args.from_release, args.to_release, args.port)

414
software/software/utilities/utils.py Normal file
View File

@ -0,0 +1,414 @@
#
# Copyright (c) 2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import keyring
import os
import psycopg2
from psycopg2.extras import RealDictCursor
import subprocess
import tempfile
import yaml
# WARNING: The first controller upgrade is done before any puppet manifests
# have been applied, so only the static entries from tsconfig can be used.
# (the platform.conf file will not have been updated with dynamic values).
from software.utilities.constants import PLATFORM_PATH
from software.utilities.constants import KEYRING_PERMDIR
from software.utilities import constants
from oslo_log import log
LOG = log.getLogger(__name__)
DB_CONNECTION = "postgresql://%s:%s@127.0.0.1/%s\n"
KUBERNETES_CONF_PATH = "/etc/kubernetes"
KUBERNETES_ADMIN_CONF_FILE = "admin.conf"
PLATFORM_LOG = '/var/log/platform.log'
ERROR_FILE = '/tmp/upgrade_fail_msg'
# well-known default domain name
DEFAULT_DOMAIN_NAME = 'Default'
# Migration script actions
ACTION_START = "start"
ACTION_MIGRATE = "migrate"
ACTION_ACTIVATE = "activate"
def execute_migration_scripts(from_release, to_release, action, port=None,
migration_script_dir="/etc/upgrade.d"):
"""Execute migration scripts with an action:
start: Prepare for upgrade on release N side. Called during
"system upgrade-start".
migrate: Perform data migration on release N+1 side. Called while
system data migration is taking place.
"""
LOG.info("Executing migration scripts with from_release: %s, "
"to_release: %s, action: %s" % (from_release, to_release, action))
# Get a sorted list of all the migration scripts
# Exclude any files that can not be executed, including .pyc and .pyo files
files = [f for f in os.listdir(migration_script_dir)
if os.path.isfile(os.path.join(migration_script_dir, f)) and
os.access(os.path.join(migration_script_dir, f), os.X_OK)]
# From file name, get the number to sort the calling sequence,
# abort when the file name format does not follow the pattern
# "nnn-*.*", where "nnn" string shall contain only digits, corresponding
# to a valid unsigned integer (first sequence of characters before "-")
try:
files.sort(key=lambda x: int(x.split("-")[0]))
except Exception:
LOG.exception("Migration script sequence validation failed, invalid "
"file name format")
raise
MSG_SCRIPT_FAILURE = "Migration script %s failed with returncode %d" \
"Script output:\n%s"
# Execute each migration script
for f in files:
migration_script = os.path.join(migration_script_dir, f)
try:
LOG.info("Executing migration script %s" % migration_script)
cmdline = [migration_script, from_release, to_release, action]
if port is not None:
cmdline.append(port)
ret = subprocess.run(cmdline,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
text=True, check=True)
if ret.returncode != 0:
script_output = ret.stdout.splitlines()
output_list = []
for item in script_output:
if item not in output_list:
output_list.append(item)
output_script = "\n".join(output_list)
msg = MSG_SCRIPT_FAILURE % (migration_script,
ret.returncode,
output_script)
LOG.error(msg)
raise Exception(msg)
except subprocess.CalledProcessError as e:
# log script output if script executed but failed.
LOG.error(MSG_SCRIPT_FAILURE %
(migration_script, e.returncode, e.output))
# Abort when a migration script fails
raise
except Exception as e:
# log exception if script not executed.
LOG.exception(e)
raise
def get_db_connection(hiera_db_records, database):
username = hiera_db_records[database]['username']
password = hiera_db_records[database]['password']
return "postgresql://%s:%s@%s/%s" % (
username, password, 'localhost', database)
def get_password_from_keyring(service, username):
"""Retrieve password from keyring"""
password = ""
os.environ["XDG_DATA_HOME"] = KEYRING_PERMDIR
try:
password = keyring.get_password(service, username)
except Exception as e:
LOG.exception("Received exception when attempting to get password "
"for service %s, username %s: %s" %
(service, username, e))
raise
finally:
del os.environ["XDG_DATA_HOME"]
return password
def get_upgrade_token(from_release,
config,
secure_config):
# Get the system hiera data from the from release
from_hiera_path = os.path.join(PLATFORM_PATH, "puppet", from_release,
"hieradata")
system_file = os.path.join(from_hiera_path, "system.yaml")
with open(system_file, 'r') as s_file:
system_config = yaml.load(s_file, Loader=yaml.Loader)
# during a data-migration, keystone is running
# on the controller UNIT IP, however the service catalog
# that was migrated from controller-0 since lists the
# floating controller IP. Keystone operations that use
# the AUTH URL will hit this service URL and fail,
# therefore we have to issue an Upgrade token for
# all Keystone operations during an Upgrade. This token
# will allow us to circumvent the service catalog entry, by
# providing a bypass endpoint.
keystone_upgrade_url = "http://{}:5000/{}".format(
'127.0.0.1',
system_config['openstack::keystone::params::api_version'])
admin_user_domain = system_config.get(
'platform::client::params::admin_user_domain')
if admin_user_domain is None:
# This value wasn't present in R2. So may be missing in upgrades from
# that release
LOG.info("platform::client::params::admin_user_domain key not found. "
"Using Default.")
admin_user_domain = DEFAULT_DOMAIN_NAME
admin_project_domain = system_config.get(
'platform::client::params::admin_project_domain')
if admin_project_domain is None:
# This value wasn't present in R2. So may be missing in upgrades from
# that release
LOG.info("platform::client::params::admin_project_domain key not "
"found. Using Default.")
admin_project_domain = DEFAULT_DOMAIN_NAME
admin_password = get_password_from_keyring("CGCS", "admin")
admin_username = system_config.get(
'platform::client::params::admin_username')
# the upgrade token command
keystone_upgrade_token = (
"openstack "
"--os-username {} "
"--os-password '{}' "
"--os-auth-url {} "
"--os-project-name admin "
"--os-user-domain-name {} "
"--os-project-domain-name {} "
"--os-interface internal "
"--os-identity-api-version 3 "
"token issue -c id -f value".format(
admin_username,
admin_password,
keystone_upgrade_url,
admin_user_domain,
admin_project_domain
))
config.update({
'openstack::keystone::upgrade::upgrade_token_file':
'/etc/keystone/upgrade_token',
'openstack::keystone::upgrade::url': keystone_upgrade_url
})
secure_config.update({
'openstack::keystone::upgrade::upgrade_token_cmd':
keystone_upgrade_token,
})
def get_upgrade_data(from_release,
system_config,
secure_config):
"""Retrieve required data from the from-release, update system_config
and secure_config with them.
This function is needed for adding new service account and endpoints
during upgrade.
"""
# Get the system hiera data from the from release
from_hiera_path = os.path.join(PLATFORM_PATH, "puppet", from_release,
"hieradata")
system_file = os.path.join(from_hiera_path, "system.yaml")
with open(system_file, 'r') as s_file:
system_config_from_release = yaml.load(s_file, Loader=yaml.Loader)
# Get keystone region
keystone_region = system_config_from_release.get(
'keystone::endpoint::region')
system_config.update({
'platform::client::params::identity_region': keystone_region,
# Retrieve keystone::auth::region from the from-release for the new
# service.
# 'newservice::keystone::auth::region': keystone_region,
})
# Generate password for the new service
# password = sysinv_utils.generate_random_password(16)
secure_config.update({
# Generate and set the keystone::auth::password for the new service.
# 'newservice::keystone::auth::password': password,
})
def add_upgrade_entries_to_hiera_data(from_release):
"""Adds upgrade entries to the hiera data """
filename = 'static.yaml'
secure_filename = 'secure_static.yaml'
path = constants.HIERADATA_PERMDIR
# Get the hiera data for this release
filepath = os.path.join(path, filename)
with open(filepath, 'r') as c_file:
config = yaml.load(c_file, Loader=yaml.Loader)
secure_filepath = os.path.join(path, secure_filename)
with open(secure_filepath, 'r') as s_file:
secure_config = yaml.load(s_file, Loader=yaml.Loader)
# File for system.yaml
# TODO(bqian): This is needed for adding new service account and endpoints
# during upgrade.
system_filename = 'system.yaml'
system_filepath = os.path.join(path, system_filename)
# Get a token and update the config
# Below should be removed. Need to ensure during data migration
get_upgrade_token(from_release, config, secure_config)
# Get required data from the from-release and add them in system.yaml.
# We don't carry system.yaml from the from-release.
# This is needed for adding new service account and endpoints
# during upgrade.
# TODO(bqian): Below should be replaced with generating hieradata from
# migrated to-release database after "deploy host" is verified
system_config = {}
get_upgrade_data(from_release, system_config, secure_config)
# Update the hiera data on disk
try:
fd, tmppath = tempfile.mkstemp(dir=path, prefix=filename,
text=True)
with open(tmppath, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
os.close(fd)
os.rename(tmppath, filepath)
except Exception:
LOG.exception("failed to write config file: %s" % filepath)
raise
try:
fd, tmppath = tempfile.mkstemp(dir=path, prefix=secure_filename,
text=True)
with open(tmppath, 'w') as f:
yaml.dump(secure_config, f, default_flow_style=False)
os.close(fd)
os.rename(tmppath, secure_filepath)
except Exception:
LOG.exception("failed to write secure config: %s" % secure_filepath)
raise
# Add required hiera data into system.yaml.
# This is needed for adding new service account and endpoints
# during upgrade.
try:
fd, tmppath = tempfile.mkstemp(dir=path, prefix=system_filename,
text=True)
with open(tmppath, 'w') as f:
yaml.dump(system_config, f, default_flow_style=False)
os.close(fd)
os.rename(tmppath, system_filepath)
except Exception:
LOG.exception("failed to write system config: %s" % system_filepath)
raise
def apply_upgrade_manifest(controller_address):
"""Apply puppet upgrade manifest files."""
cmd = [
"/usr/local/bin/puppet-manifest-apply.sh",
constants.HIERADATA_PERMDIR,
str(controller_address),
constants.CONTROLLER,
'upgrade'
]
logfile = "/tmp/apply_manifest.log"
try:
with open(logfile, "w") as flog:
subprocess.check_call(cmd, stdout=flog, stderr=flog)
except subprocess.CalledProcessError:
msg = "Failed to execute upgrade manifest"
print(msg)
raise Exception(msg)
def get_keystone_user_id(user_name):
"""Get the a keystone user id by name"""
conn = psycopg2.connect("dbname='keystone' user='postgres'")
with conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute("SELECT user_id FROM local_user WHERE name='%s'" %
user_name)
user_id = cur.fetchone()
if user_id is not None:
return user_id['user_id']
else:
return user_id
def get_keystone_project_id(project_name):
"""Get the a keystone project id by name"""
conn = psycopg2.connect("dbname='keystone' user='postgres'")
with conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute("SELECT id FROM project WHERE name='%s'" %
project_name)
project_id = cur.fetchone()
if project_id is not None:
return project_id['id']
else:
return project_id
def get_postgres_bin():
"""Get the path to the postgres binaries"""
try:
return subprocess.check_output(
['pg_config', '--bindir']).decode().rstrip('\n')
except subprocess.CalledProcessError:
LOG.exception("Failed to get postgres bin directory.")
raise
def create_manifest_runtime_config(filename, config):
"""Write the runtime Puppet configuration to a runtime file."""
if not config:
return
try:
with open(filename, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
except Exception:
LOG.exception("failed to write config file: %s" % filename)
raise
def create_system_config():
cmd = ["/usr/bin/sysinv-puppet",
"create-system-config",
constants.HIERADATA_PERMDIR]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
msg = "Failed to update puppet hiera system config"
print(msg)
raise Exception(msg)
def create_host_config(hostname=None):
cmd = ["/usr/bin/sysinv-puppet",
"create-host-config",
constants.HIERADATA_PERMDIR]
if hostname:
cmd.append(hostname)
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
msg = "Failed to update puppet hiera host config"
print(msg)
raise Exception(msg)