From c4b74f48014969759d3783ee895f0a7b9cc68a1d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 19 Jan 2022 13:51:34 +0000 Subject: [PATCH] libvirt: deploy libvirt on the host In some cases it may be desirable to run libvirt daemon on the host. For example, when mixing host and container OS distributions. This change makes it possible to disable the nova_libvirt container, by setting kolla_enable_nova_libvirt_container to false. The stackhpc.libvirt-host role is used in order to install and configure a libvirt daemon on compute hosts when kolla_enable_nova_libvirt_container is false. Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/825357 Depends-On: https://review.opendev.org/c/openstack/kayobe-config-dev/+/829225 Depends-On: https://github.com/stackhpc/ansible-role-libvirt-host/pull/51 Story: 2009858 Task: 44495 Change-Id: I73fef63fb886a9d543d2f4231fb009523495edb3 --- ansible/compute-libvirt-host.yml | 56 +++++++ ansible/group_vars/all/compute | 51 ++++++ ansible/group_vars/all/kolla | 1 + ansible/kolla-ansible.yml | 1 + ansible/kolla-openstack.yml | 2 + ansible/roles/kolla-ansible/defaults/main.yml | 2 + .../kolla-ansible/templates/kolla/globals.yml | 4 + ansible/roles/kolla-ansible/vars/main.yml | 1 + .../roles/kolla-openstack/defaults/main.yml | 10 ++ .../molecule/enable-everything/molecule.yml | 5 +- .../molecule/enable-everything/prepare.yml | 20 +++ .../enable-everything/tests/test_default.py | 6 +- .../roles/kolla-openstack/tasks/config.yml | 2 +- ansible/roles/kolla-openstack/vars/main.yml | 21 +++ dev/functions | 45 ++++-- ...-deploy-config-compute-libvirt-on-host.yml | 56 +++++++ doc/source/configuration/reference/hosts.rst | 151 ++++++++++++++++++ doc/source/contributor/automated.rst | 5 - etc/kayobe/compute.yml | 47 ++++++ kayobe/cli/commands.py | 4 +- kayobe/tests/unit/cli/test_commands.py | 4 + .../kayobe-overcloud-base/globals.yml.j2 | 1 + .../kayobe-overcloud-base/overrides.yml.j2 | 3 + playbooks/kayobe-overcloud-base/run.yml | 4 +- .../libvirt-on-host-ff83f12923cc1f58.yaml | 7 + requirements.yml | 2 +- 26 files changed, 483 insertions(+), 28 deletions(-) create mode 100644 ansible/compute-libvirt-host.yml create mode 100644 dev/tenks-deploy-config-compute-libvirt-on-host.yml create mode 100644 releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml diff --git a/ansible/compute-libvirt-host.yml b/ansible/compute-libvirt-host.yml new file mode 100644 index 000000000..7e5501938 --- /dev/null +++ b/ansible/compute-libvirt-host.yml @@ -0,0 +1,56 @@ +--- +- name: Ensure the libvirt daemon is configured + hosts: compute + tags: + - libvirt-host + tasks: + - name: Ensure Ceph package repository is available + package: + name: "centos-release-ceph-{{ compute_libvirt_ceph_repo_release }}" + state: present + when: + - compute_libvirt_enabled | bool + - ansible_facts.distribution in ['CentOS', 'Rocky'] + - compute_libvirt_ceph_repo_install | bool + become: true + + - name: Include stackhpc.libvirt-host role + include_role: + name: stackhpc.libvirt-host + vars: + libvirt_host_libvirtd_conf: "{{ compute_libvirt_conf }}" + libvirt_host_qemu_conf: "{{ compute_qemu_conf }}" + libvirt_host_tcp_listen: "{{ not compute_libvirt_enable_tls | bool }}" + libvirt_host_tcp_listen_address: "{{ internal_net_name | net_ip }}:16509" + libvirt_host_tls_listen: "{{ compute_libvirt_enable_tls | bool }}" + libvirt_host_tls_listen_address: "{{ internal_net_name | net_ip }}:16514" + # TLS server and client certificates. + libvirt_host_tls_server_cert: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['servercert.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_server_key: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['serverkey.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_client_cert: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientcert.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_client_key: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientkey.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_cacert: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['cacert.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + lookup_params: + paths: "{{ libvirt_tls_cert_paths }}" + skip: true + # Support loading libvirt TLS certificates & keys from per-host and + # global locations. + libvirt_tls_cert_paths: >- + {{ (libvirt_tls_cert_dirs | unique | product([inventory_hostname]) | map('path_join') | list + + libvirt_tls_cert_dirs | unique | list) | list }} + libvirt_tls_cert_dirs: + - "{{ kayobe_env_config_path }}/certificates/libvirt" + - "{{ kayobe_config_path }}/certificates/libvirt" + libvirt_host_enable_efi_support: true + when: + - compute_libvirt_enabled | bool diff --git a/ansible/group_vars/all/compute b/ansible/group_vars/all/compute index dba12a943..b8b1e8161 100644 --- a/ansible/group_vars/all/compute +++ b/ansible/group_vars/all/compute @@ -161,3 +161,54 @@ compute_firewalld_default_zone: # - permanent: true # - state: enabled compute_firewalld_rules: [] + +############################################################################### +# Compute node host libvirt configuration. + +# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova +# is true and kolla_enable_nova_libvirt_container is false. +compute_libvirt_enabled: "{{ kolla_enable_nova | bool and not kolla_enable_nova_libvirt_container | bool }}" + +# A dict of default configuration options to write to +# /etc/libvirt/libvirtd.conf. +compute_libvirt_conf_default: + auth_tcp: "none" + log_level: "{{ compute_libvirtd_log_level }}" + +# A dict of additional configuration options to write to +# /etc/libvirt/libvirtd.conf. +compute_libvirt_conf_extra: {} + +# A dict of configuration options to write to /etc/libvirt/libvirtd.conf. +# Default is a combination of compute_libvirt_conf_default and +# compute_libvirt_conf_extra. +compute_libvirt_conf: "{{ compute_libvirt_conf_default | combine(compute_libvirt_conf_extra) }}" + +# Numerical log level for libvirtd. Default is 3. +compute_libvirtd_log_level: 3 + +# A dict of default configuration options to write to +# /etc/libvirt/qemu.conf. +compute_qemu_conf_default: + max_files: 32768 + max_processes: 131072 + +# A dict of additional configuration options to write to +# /etc/libvirt/qemu.conf. +compute_qemu_conf_extra: {} + +# A dict of configuration options to write to /etc/libvirt/qemu.conf. +# Default is a combination of compute_qemu_conf_default and +# compute_qemu_conf_extra. +compute_qemu_conf: "{{ compute_qemu_conf_default | combine(compute_qemu_conf_extra) }}" + +# Whether to enable a libvirt TLS listener. Default is false. +compute_libvirt_enable_tls: false + +# Whether to install a Ceph package repository on CentOS and Rocky hosts. +# Default is true. +compute_libvirt_ceph_repo_install: true + +# Ceph package repository release to install on CentOS and Rocky hosts when +# compute_libvirt_ceph_repo_install is true. Default is 'pacific'. +compute_libvirt_ceph_repo_release: pacific diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index f10fd789c..ede0e5b33 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -553,6 +553,7 @@ kolla_enable_murano: "no" kolla_enable_neutron_mlnx: "no" kolla_enable_neutron_provider_networks: "no" kolla_enable_neutron_sriov: "no" +kolla_enable_nova_libvirt_container: "yes" kolla_enable_octavia: "no" kolla_enable_openvswitch: "{{ kolla_enable_neutron | bool }}" kolla_enable_ovn: "no" diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml index f2ccc3432..a469e474d 100644 --- a/ansible/kolla-ansible.yml +++ b/ansible/kolla-ansible.yml @@ -103,6 +103,7 @@ kolla_inspector_netmask: "{{ inspection_net_name | net_mask }}" kolla_inspector_default_gateway: "{{ inspection_net_name | net_inspection_gateway or inspection_net_name | net_gateway }}" kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}" + kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}" kolla_enable_host_ntp: false docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}" kolla_globals_paths_extra: diff --git a/ansible/kolla-openstack.yml b/ansible/kolla-openstack.yml index 36cf4daa1..6941067ee 100644 --- a/ansible/kolla-openstack.yml +++ b/ansible/kolla-openstack.yml @@ -249,3 +249,5 @@ kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}" kolla_extra_zookeeper: "{{ kolla_extra_config.zookeeper | default }}" kolla_extra_config_path: "{{ kayobe_env_config_path }}/kolla/config" + kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}" + kolla_nova_libvirt_certificates_src: "{{ kayobe_env_config_path }}/certificates/libvirt" diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index 8a3ff899e..9e0774482 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -236,6 +236,8 @@ kolla_openstack_logging_debug: # controllers. kolla_nova_compute_ironic_host: +kolla_libvirt_tls: + ############################################################################### # Extra free-form configuraton. diff --git a/ansible/roles/kolla-ansible/templates/kolla/globals.yml b/ansible/roles/kolla-ansible/templates/kolla/globals.yml index 8e0784541..04d5e33b3 100644 --- a/ansible/roles/kolla-ansible/templates/kolla/globals.yml +++ b/ansible/roles/kolla-ansible/templates/kolla/globals.yml @@ -393,6 +393,10 @@ enable_{{ feature_flag }}: {{ hostvars[inventory_hostname]['kolla_enable_' ~ fea # Valid options are [ none, novnc, spice, rdp ] #nova_console: "novnc" +{% if kolla_libvirt_tls is not none %} +libvirt_tls: {{ kolla_libvirt_tls | bool }} +{% endif %} + ################# # Hyper-V options ################# diff --git a/ansible/roles/kolla-ansible/vars/main.yml b/ansible/roles/kolla-ansible/vars/main.yml index 1f3fd0d5e..968c2222b 100644 --- a/ansible/roles/kolla-ansible/vars/main.yml +++ b/ansible/roles/kolla-ansible/vars/main.yml @@ -184,6 +184,7 @@ kolla_feature_flags: - nova - nova_fake - nova_horizon_policy_file + - nova_libvirt_container - nova_serialconsole_proxy - nova_ssh - octavia diff --git a/ansible/roles/kolla-openstack/defaults/main.yml b/ansible/roles/kolla-openstack/defaults/main.yml index ae32392a7..6d2ad45de 100644 --- a/ansible/roles/kolla-openstack/defaults/main.yml +++ b/ansible/roles/kolla-openstack/defaults/main.yml @@ -447,9 +447,19 @@ kolla_extra_neutron_ml2: # Whether to enable Nova. kolla_enable_nova: +# Whether to enable Nova libvirt container. +kolla_enable_nova_libvirt_container: + # Free form extra configuration to append to nova.conf. kolla_extra_nova: +# Whether libvirt TLS is enabled. +kolla_libvirt_tls: + +# Directory containing libvirt certificates for nova-compute when running +# libvirt on the host. +kolla_nova_libvirt_certificates_src: + ############################################################################### # Octavia configuration. diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml b/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml index 14b0bbc86..634455c19 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml @@ -15,7 +15,7 @@ provisioner: inventory: group_vars: all: - kolla_extra_config_path: + kolla_extra_config_path: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/kolla/config kolla_enable_aodh: true kolla_extra_aodh: | [extra-aodh.conf] @@ -116,9 +116,12 @@ provisioner: [extra-ml2_conf.ini] foo=bar kolla_enable_nova: true + kolla_enable_nova_libvirt_container: false kolla_extra_nova: | [extra-nova.conf] foo=bar + kolla_libvirt_tls: true + kolla_nova_libvirt_certificates_src: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/nova-libvirt/certificates kolla_enable_octavia: true kolla_extra_octavia: | [extra-octavia.conf] diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml b/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml index d78cc6940..8514e90f3 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml @@ -25,3 +25,23 @@ with_items: - "{{ kolla_inspector_ipa_kernel_path }}" - "{{ kolla_inspector_ipa_ramdisk_path }}" + + - name: Ensure nova libvirt certificates directory exists + local_action: + module: file + path: "{{ kolla_nova_libvirt_certificates_src }}" + state: directory + + # NOTE(mgoddard): Previously we were creating empty files for the kernel + # and ramdisk, but this was found to cause ansible to hang on recent + # versions of docker. Using non-empty files seems to resolve the issue. + # See https://github.com/ansible/ansible/issues/36725. + - name: Ensure nova libvirt certificates exist + local_action: + module: copy + content: fake cert + dest: "{{ kolla_nova_libvirt_certificates_src }}/{{ item }}" + with_items: + - "cacert.pem" + - "clientcert.pem" + - "clientkey.pem" diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py b/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py index 6d5f67fe5..0badbfefd 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py @@ -50,6 +50,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 'murano', 'neutron', 'nova', + 'nova/nova-libvirt', 'octavia', 'placement', 'prometheus', @@ -100,7 +101,10 @@ def test_service_ini_file(host, path): @pytest.mark.parametrize( 'path', ['ironic/ironic-agent.initramfs', - 'ironic/ironic-agent.kernel']) + 'ironic/ironic-agent.kernel', + 'nova/nova-libvirt/cacert.pem', + 'nova/nova-libvirt/clientcert.pem', + 'nova/nova-libvirt/clientkey.pem']) def test_service_non_ini_file(host, path): # TODO(mgoddard): Check config file contents. path = os.path.join('/etc/kolla/config', path) diff --git a/ansible/roles/kolla-openstack/tasks/config.yml b/ansible/roles/kolla-openstack/tasks/config.yml index 5123bcda0..734b5a88b 100644 --- a/ansible/roles/kolla-openstack/tasks/config.yml +++ b/ansible/roles/kolla-openstack/tasks/config.yml @@ -80,6 +80,7 @@ recurse: true with_items: "{{ kolla_openstack_custom_config }}" register: find_src_result + delegate_to: localhost - name: Find previously generated extra configuration files find: @@ -91,7 +92,6 @@ - name: Ensure extra configuration parent directories are present file: path: "{{ item.0.item.dest }}/{{ item.1.path | relpath(item.0.item.src) | dirname }}" - recurse: true state: directory mode: 0750 with_subelements: diff --git a/ansible/roles/kolla-openstack/vars/main.yml b/ansible/roles/kolla-openstack/vars/main.yml index e7252a137..8033b354f 100644 --- a/ansible/roles/kolla-openstack/vars/main.yml +++ b/ansible/roles/kolla-openstack/vars/main.yml @@ -178,6 +178,27 @@ kolla_openstack_custom_config: dest: "{{ kolla_node_custom_config_path }}/nova" patterns: "*" enabled: "{{ kolla_enable_nova }}" + # Nova. + - src: "{{ kolla_nova_libvirt_certificates_src }}" + dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt" + patterns: + - clientcert.pem + - clientkey.pem + - cacert.pem + enabled: "{{ kolla_enable_nova | bool and kolla_libvirt_tls | bool }}" + untemplated: + - clientcert.pem + - clientkey.pem + - cacert.pem + - src: "{{ kolla_nova_libvirt_certificates_src }}" + dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt" + patterns: + - servercert.pem + - serverkey.pem + enabled: "{{ kolla_enable_nova | bool and kolla_enable_nova_libvirt_container | bool and kolla_libvirt_tls | bool }}" + untemplated: + - servercert.pem + - serverkey.pem # Octavia. - src: "{{ kolla_extra_config_path }}/octavia" dest: "{{ kolla_node_custom_config_path }}/octavia" diff --git a/dev/functions b/dev/functions index 3ece5f23f..436d33e57 100644 --- a/dev/functions +++ b/dev/functions @@ -10,12 +10,8 @@ set -o pipefail function config_defaults { # Set default values for kayobe development configuration. - # Try to detect if we are running in a vagrant VM. - if [[ -e /vagrant ]]; then - KAYOBE_SOURCE_PATH_DEFAULT=/vagrant - else - KAYOBE_SOURCE_PATH_DEFAULT="$(pwd)" - fi + PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + KAYOBE_SOURCE_PATH_DEFAULT="$(dirname ${PARENT})" # Path to the kayobe source code repository. Typically this will be the # Vagrant shared directory. @@ -392,18 +388,26 @@ function overcloud_deploy { control_host_bootstrap - echo "Configuring the controller host" - run_kayobe overcloud host configure - - # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname - # resolves to IP address of API interface for RabbitMQ. This seems to be - # required since https://review.openstack.org/#/c/584427 was merged. - echo "Workaround: upgrading the controller host" - run_kayobe overcloud host upgrade - if [[ ${KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES} = 1 ]]; then echo "Generate TLS certificates" - run_kayobe kolla ansible run certificates --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates + run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/kolla-ansible.yml -t config + # NOTE(mgoddard): There is a chicken and egg when generating libvirt + # TLS certificates using the kolla-ansible certificates command, and + # host libvirt. The certificates command needs to be able to gather + # facts for all hosts, but since the host configure step hasn't been + # run, we don't have SSH or the kolla user configured yet. However, we + # can't run host configure without the libvirt TLS certificates. + # Workaround: add the host to SSH known hosts and SSH as $USER. + run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/ssh-known-host.yml -l overcloud + + # Avoid populating the fact cache with this weird setup. + export ANSIBLE_CACHE_PLUGIN=memory + run_kayobe kolla ansible run certificates \ + --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates \ + --kolla-extra ansible_user=$USER \ + --kolla-extra ansible_python_interpreter=/usr/bin/python3 + unset ANSIBLE_CACHE_PLUGIN + # Add CA cert to trust store. ca_cert=${KAYOBE_CONFIG_PATH}/kolla/certificates/ca/root.crt if [[ -e /etc/debian_version ]]; then @@ -417,6 +421,15 @@ function overcloud_deploy { fi fi + echo "Configuring the controller host" + run_kayobe overcloud host configure + + # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname + # resolves to IP address of API interface for RabbitMQ. This seems to be + # required since https://review.openstack.org/#/c/584427 was merged. + echo "Workaround: upgrading the controller host" + run_kayobe overcloud host upgrade + # Note: This must currently be before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the # built deploy images. diff --git a/dev/tenks-deploy-config-compute-libvirt-on-host.yml b/dev/tenks-deploy-config-compute-libvirt-on-host.yml new file mode 100644 index 000000000..d8cd1a14f --- /dev/null +++ b/dev/tenks-deploy-config-compute-libvirt-on-host.yml @@ -0,0 +1,56 @@ +--- +# This file holds the config given to Tenks when running `tenks-deploy.sh`. It +# assumes the existence of the bridge `breth1`. + +node_types: + type0: + memory_mb: 1024 + vcpus: 1 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 4GiB + physical_networks: + - physnet1 + console_log_enabled: true + # We seem to hit issues with missing cpu features in CI as a result of using host-model, e.g: + # https://zuul.opendev.org/t/openstack/build/02c33ab51664419a88a5a54ad22852a9/log/primary/system_logs/libvirt/qemu/tk0.txt.gz#38 + cpu_mode: + +specs: + - type: type0 + count: 2 + ironic_config: + resource_class: test-rc + network_interface: flat + +nova_flavors: + - resource_class: test-rc + node_type: type0 + +physnet_mappings: + physnet1: breth1 + +deploy_kernel: ipa.kernel +deploy_ramdisk: ipa.initramfs + +default_boot_mode: "bios" + +# Use the libvirt daemon deployed by Kayobe. Tenks will install libvirt client +# packages. +libvirt_host_install_daemon: false + +# Configure AppArmor for the pool on Ubuntu. +libvirt_host_configure_apparmor: true + +# Nested virtualisation is not working well in CI currently. Force the use of +# QEMU. +libvirt_vm_engine: "qemu" + +# QEMU may not be installed on the host, so set the path and avoid +# autodetection. +libvirt_vm_emulator: "{% if ansible_facts.os_family == 'RedHat' %}/usr/libexec/qemu-kvm{% else %}/usr/bin/qemu-system-x86_64{% endif %}" + +# Specify a log path in the kolla_logs Docker volume. It is accessible on the +# host at the same path. +libvirt_vm_default_console_log_dir: "/var/log/kolla/tenks" diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst index 710d0e195..7cda51e2b 100644 --- a/doc/source/configuration/reference/hosts.rst +++ b/doc/source/configuration/reference/hosts.rst @@ -1044,3 +1044,154 @@ Ansible's containers do), but may be necessary when building images. Docker's live restore feature can be configured via ``docker_daemon_live_restore``, although it is disabled by default due to issues observed. + +Compute libvirt daemon +====================== +*tags:* + | ``libvirt-host`` + +.. note:: + + This section is about the libvirt daemon on compute nodes, as opposed to the + seed hypervisor. + +Since Yoga, Kayobe provides support for deploying and configuring a libvirt +host daemon, as an alternative to the ``nova_libvirt`` container support by +Kolla Ansible. The host daemon is not used by default, but it is possible to +enable it by setting ``kolla_enable_nova_libvirt_container`` to ``false`` in +``$KAYOBE_CONFIG_PATH/kolla.yml``. + +Migration of hosts from a containerised libvirt to host libvirt is currently +not supported. + +The following options are available in ``$KAYOBE_CONFIG_PATH/compute.yml`` and +are relevant only when using the libvirt daemon rather than the +``nova_libvirt`` container: + +``compute_libvirt_enabled`` + Whether to enable a host libvirt daemon. Default is true if + ``kolla_enable_nova`` is ``true`` and + ``kolla_enable_nova_libvirt_container`` is ``false``. +``compute_libvirt_conf_default`` + A dict of default configuration options to write to + ``/etc/libvirt/libvirtd.conf``. +``compute_libvirt_conf_extra`` + A dict of additional configuration options to write to + ``/etc/libvirt/libvirtd.conf``. +``compute_libvirt_conf`` + A dict of configuration options to write to ``/etc/libvirt/libvirtd.conf``. + Default is a combination of ``compute_libvirt_conf_default`` and + ``compute_libvirt_conf_extra``. +``compute_libvirtd_log_level`` + Numerical log level for libvirtd. Default is 3. +``compute_qemu_conf_default`` + A dict of default configuration options to write to + ``/etc/libvirt/qemu.conf``. +``compute_qemu_conf_extra`` + A dict of additional configuration options to write to + ``/etc/libvirt/qemu.conf``. +``compute_qemu_conf`` + A dict of configuration options to write to ``/etc/libvirt/qemu.conf``. + Default is a combination of ``compute_qemu_conf_default`` and + ``compute_qemu_conf_extra``. +``compute_libvirt_enable_tls`` + Whether to enable a libvirt TLS listener. Default is false. +``compute_libvirt_ceph_repo_install`` + Whether to install a Ceph package repository on CentOS and Rocky hosts. + Default is ``true``. +``compute_libvirt_ceph_repo_release`` + Ceph package repository release to install on CentOS and Rocky hosts when + ``compute_libvirt_ceph_repo_install`` is ``true``. Default is ``pacific``. + +Example: custom libvirtd.conf +----------------------------- + +To customise the libvirt daemon log output to send level 3 to the journal: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_libvirt_conf_extra: + log_outputs: "3:journald" + +Example: custom qemu.conf +------------------------- + +To customise QEMU to avoid adding timestamps to logs: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_qemu_conf_extra: + log_timestamp: 0 + +Example: enabling libvirt TLS listener +-------------------------------------- + +To enable the libvirt TLS listener: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_libvirt_enable_tls: true + +When the TLS listener is enabled, it is necessary to provide client, server and +CA certificates. The following files should be provided: + +``cacert.pem`` + CA certificate used to sign client and server certificates. +``clientcert.pem`` + Client certificate. +``clientkey.pem`` + Client key. +``servercert.pem`` + Server certificate. +``serverkey.pem`` + Server key. + +It is recommended to encrypt the key files using Ansible Vault. + +The following paths are searched for these files: + +* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/{{ inventory_hostname }}/`` +* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/`` + +In this way, certificates may be generated for each host, or shared using +wildcard certificates. + +If using Kayobe environments, certificates in the environment take precedence. + +Kayobe makes the CA certificate and client certificate and key available to +Kolla Ansible, for use by the ``nova_compute`` service. + +Example: disabling Ceph repository installation +----------------------------------------------- + +On CentOS and Rocky hosts, a CentOS Storage SIG Ceph repository is installed +that provides more recent Ceph libraries than those available in CentOS/Rocky +AppStream. This may be necessary when using Ceph for Cinder volumes or Nova +ephemeral block devices. In some cases, such as when using local package +mirrors, the upstream repository may not be appropriate. The installation of +the repository may be disabled as follows: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_libvirt_ceph_repo_install: false + +Example: installing additional packages +--------------------------------------- + +In some cases it may be useful to install additional packages on compute hosts +for use by libvirt. The `stackhpc.libvirt-host +`__ Ansible role supports +this via the ``libvirt_host_extra_daemon_packages`` variable. The variable +should be defined via group variables in the Ansible inventory, to avoid +applying the change to the seed hypervisor. For example, to install the +``trousers`` package used for accessing TPM hardware: + +.. code-block:: yaml + :caption: ``inventory/group_vars/compute/libvirt`` + + libvirt_host_extra_daemon_packages: + - trousers diff --git a/doc/source/contributor/automated.rst b/doc/source/contributor/automated.rst index 8e71d1cf7..c0e7cc657 100644 --- a/doc/source/contributor/automated.rst +++ b/doc/source/contributor/automated.rst @@ -131,11 +131,6 @@ For a control plane with Ironic enabled, a "bare metal" instance can be deployed. We can use the `Tenks `__ project to create fake bare metal nodes. -On Ubuntu, the ``nova_libvirt`` image does not contain the ``qemu-utils`` -package necessary for image operations used by Tenks. Install it as follows:: - - sudo docker exec -u root nova_libvirt bash -c 'apt update && apt -y install qemu-utils' - Clone the tenks repository:: git clone https://opendev.org/openstack/tenks.git diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index cd0ceb247..b1d8d6562 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -143,6 +143,53 @@ # - state: enabled #compute_firewalld_rules: +############################################################################### +# Compute node host libvirt configuration. + +# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova +# is true and kolla_enable_nova_libvirt_container is false. +#compute_libvirt_enabled: + +# A dict of default configuration options to write to +# /etc/libvirt/libvirtd.conf. +#compute_libvirt_conf_default: + +# A dict of additional configuration options to write to +# /etc/libvirt/libvirtd.conf. +#compute_libvirt_conf_extra: + +# A dict of configuration options to write to /etc/libvirt/libvirtd.conf. +# Default is a combination of compute_libvirt_conf_default and +# compute_libvirt_conf_extra. +#compute_libvirt_conf: + +# Numerical log level for libvirtd. Default is 3. +#compute_libvirtd_log_level: + +# A dict of default configuration options to write to +# /etc/libvirt/qemu.conf. +#compute_qemu_conf_default: + +# A dict of additional configuration options to write to +# /etc/libvirt/qemu.conf. +#compute_qemu_conf_extra: + +# A dict of configuration options to write to /etc/libvirt/qemu.conf. +# Default is a combination of compute_qemu_conf_default and +# compute_qemu_conf_extra. +#compute_qemu_conf: + +# Whether to enable a libvirt TLS listener. Default is false. +#compute_libvirt_enable_tls: + +# Whether to install a Ceph package repository on CentOS and Rocky hosts. +# Default is true. +#compute_libvirt_ceph_repo_install: + +# Ceph package repository release to install on CentOS and Rocky hosts when +# compute_libvirt_ceph_repo_install is true. Default is 'pacific'. +#compute_libvirt_ceph_repo_release: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 788d493b5..325a67aea 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -1125,6 +1125,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Optionally, create a virtualenv for kolla-ansible. * Configure a user account for kolla-ansible. * Configure Docker engine. + * Configure libvirt. """ def get_parser(self, prog_name): @@ -1157,7 +1158,8 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers") # Further kayobe playbooks. - playbooks = _build_playbook_list("docker", "swift-block-devices") + playbooks = _build_playbook_list( + "docker", "swift-block-devices", "compute-libvirt-host") self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index dbb402e1d..5986de76b 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -1322,6 +1322,8 @@ class TestCase(unittest.TestCase): utils.get_data_files_path("ansible", "docker.yml"), utils.get_data_files_path( "ansible", "swift-block-devices.yml"), + utils.get_data_files_path( + "ansible", "compute-libvirt-host.yml"), ], limit="overcloud", ), @@ -1376,6 +1378,8 @@ class TestCase(unittest.TestCase): utils.get_data_files_path("ansible", "docker.yml"), utils.get_data_files_path( "ansible", "swift-block-devices.yml"), + utils.get_data_files_path( + "ansible", "compute-libvirt-host.yml"), ], limit="overcloud", ), diff --git a/playbooks/kayobe-overcloud-base/globals.yml.j2 b/playbooks/kayobe-overcloud-base/globals.yml.j2 index 7de112f61..58c4c7ec9 100644 --- a/playbooks/kayobe-overcloud-base/globals.yml.j2 +++ b/playbooks/kayobe-overcloud-base/globals.yml.j2 @@ -22,4 +22,5 @@ kolla_enable_tls_backend: "yes" openstack_cacert: "/etc/pki/tls/certs/ca-bundle.crt" kolla_admin_openrc_cacert: "/etc/pki/tls/certs/ca-bundle.crt" libvirt_tls: "yes" +certificates_libvirt_output_dir: "{% raw %}{{ kayobe_env_config_path }}{% endraw %}/certificates/libvirt" {% endif %} diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2 index 58462fb52..864e29b96 100644 --- a/playbooks/kayobe-overcloud-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2 @@ -42,6 +42,9 @@ kolla_ironic_default_boot_interface: ipxe {% endif %} {% if tls_enabled %} +kolla_enable_nova_libvirt_container: false +compute_libvirt_enable_tls: true + kolla_enable_tls_external: "yes" kolla_enable_tls_internal: "yes" diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml index 3d77e86aa..350b47dfd 100644 --- a/playbooks/kayobe-overcloud-base/run.yml +++ b/playbooks/kayobe-overcloud-base/run.yml @@ -3,6 +3,8 @@ environment: KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES: "{{ tls_enabled | ternary(1, 0) }}" + # TODO(mgoddard): Remove this when libvirt on host is used by default. + TENKS_CONFIG_PATH: "dev/tenks-deploy-config-compute{% if tls_enabled %}-libvirt-on-host{% endif %}.yml" tasks: - name: Ensure overcloud is deployed shell: @@ -18,8 +20,6 @@ executable: /bin/bash - name: Perform testing of the virtualized machines - # We must do this before tenks-deploy as that will stop the nova_libvirt - # container shell: cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm chdir: "{{ kayobe_src_dir }}" diff --git a/releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml b/releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml new file mode 100644 index 000000000..a16a9cfd2 --- /dev/null +++ b/releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds support for running a libvirt daemon on the host, rather than in a + container. This is done by setting ``kolla_enable_nova_libvirt_container`` + to ``false``. See `story 2009858 + `__ for details. diff --git a/requirements.yml b/requirements.yml index 4851b821b..86c765150 100644 --- a/requirements.yml +++ b/requirements.yml @@ -32,7 +32,7 @@ roles: - src: stackhpc.grafana-conf version: 1.1.1 - src: stackhpc.libvirt-host - version: v1.8.3 + version: v1.10.0 - src: stackhpc.libvirt-vm version: v1.14.2 - src: stackhpc.luks