From c9c0019d7e66b21be82974278311925300acd594 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 24 Mar 2022 16:34:01 +0000 Subject: [PATCH] Use jinja2.pass_context instead of contextfilter The contextfilter decorator was deprecated in jinja2 3.0.0, and has been dropped in 3.1.0. This results in the following warning, and failed attempts to use filters: [WARNING]: Skipping plugin (networks.py) as it seems to be invalid: module 'jinja2' has no attribute 'contextfilter' This change switches to use the pass_context decorator. The minimum version of Jinja2 is raised to 3 to ensure pass_context is present. This change also includes some changes to address issues with image builds in CI, caused by CentOS Scream. 1. disable IPA image builds in seed deploy jobs IPA image builds will be split out into a separate job. For now, disable them. 2. disable overcloud host image builds in seed deploy jobs Overcloud host image builds will be split out into a separate job. For now, disable them. Depends-On: https://review.opendev.org/c/openstack/kayobe/+/835279 Change-Id: If657bf5b0117812d3c53942464cc41cf86cc8ad5 --- kayobe/plugins/filter/networkd.py | 6 +- kayobe/plugins/filter/networks.py | 76 +++++++++---------- .../action/test_kolla_ansible_host_vars.py | 6 +- .../kayobe-seed-base/bifrost-overrides.yml.j2 | 7 ++ playbooks/kayobe-seed-base/overrides.yml.j2 | 8 -- playbooks/kayobe-seed-base/pre.yml | 6 ++ .../jinja2-pass-context-fecf00f23e413393.yaml | 4 + requirements.txt | 1 + requirements.yml | 2 +- 9 files changed, 63 insertions(+), 53 deletions(-) create mode 100644 playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 create mode 100644 releasenotes/notes/jinja2-pass-context-fecf00f23e413393.yaml diff --git a/kayobe/plugins/filter/networkd.py b/kayobe/plugins/filter/networkd.py index 3abb742c4..f07c2eb4b 100644 --- a/kayobe/plugins/filter/networkd.py +++ b/kayobe/plugins/filter/networkd.py @@ -496,7 +496,7 @@ def _add_to_result(result, prefix, device, config): result[key] = config -@jinja2.contextfilter +@jinja2.pass_context def networkd_netdevs(context, names, inventory_hostname=None): """Return a dict representation of networkd NetDev configuration. @@ -546,7 +546,7 @@ def networkd_netdevs(context, names, inventory_hostname=None): return result -@jinja2.contextfilter +@jinja2.pass_context def networkd_links(context, names, inventory_hostname=None): """Return a dict representation of networkd link configuration. @@ -562,7 +562,7 @@ def networkd_links(context, names, inventory_hostname=None): return {} -@jinja2.contextfilter +@jinja2.pass_context def networkd_networks(context, names, inventory_hostname=None): """Return a dict representation of networkd network configuration. diff --git a/kayobe/plugins/filter/networks.py b/kayobe/plugins/filter/networks.py index c420675fd..29c6018e4 100644 --- a/kayobe/plugins/filter/networks.py +++ b/kayobe/plugins/filter/networks.py @@ -141,25 +141,25 @@ def get_vlan_parent(device, vlan): return re.sub(r'\.{}$'.format(vlan), '', device) -@jinja2.contextfilter +@jinja2.pass_context def net_attr(context, name, attr, inventory_hostname=None): var_name = "%s_%s" % (name, attr) return utils.get_hostvar(context, var_name, inventory_hostname) def _make_attr_filter(attr): - @jinja2.contextfilter + @jinja2.pass_context def func(context, name, inventory_hostname=None): return net_attr(context, name, attr, inventory_hostname) return func -@jinja2.contextfilter +@jinja2.pass_context def net_vip_address(context, name, inventory_hostname=None): return net_attr(context, name, 'vip_address', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_ip(context, name, inventory_hostname=None): ips = net_attr(context, name, 'ips', inventory_hostname) if ips: @@ -169,56 +169,56 @@ def net_ip(context, name, inventory_hostname=None): return ips.get(inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_interface(context, name, inventory_hostname=None): return net_attr(context, name, 'interface', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_no_ip(context, name, inventory_hostname=None): return net_attr(context, name, 'no_ip', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_cidr(context, name, inventory_hostname=None): return net_attr(context, name, 'cidr', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_mask(context, name, inventory_hostname=None): cidr = net_cidr(context, name, inventory_hostname) return str(netaddr.IPNetwork(cidr).netmask) if cidr is not None else None -@jinja2.contextfilter +@jinja2.pass_context def net_prefix(context, name, inventory_hostname=None): cidr = net_cidr(context, name, inventory_hostname) return str(netaddr.IPNetwork(cidr).prefixlen) if cidr is not None else None -@jinja2.contextfilter +@jinja2.pass_context def net_gateway(context, name, inventory_hostname=None): return net_attr(context, name, 'gateway', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_allocation_pool_start(context, name, inventory_hostname=None): return net_attr(context, name, 'allocation_pool_start', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_allocation_pool_end(context, name, inventory_hostname=None): return net_attr(context, name, 'allocation_pool_end', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_inspection_allocation_pool_start(context, name, inventory_hostname=None): return net_attr(context, name, 'inspection_allocation_pool_start', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_inspection_allocation_pool_end(context, name, inventory_hostname=None): return net_attr(context, name, 'inspection_allocation_pool_end', inventory_hostname) @@ -227,13 +227,13 @@ def net_inspection_allocation_pool_end(context, name, inventory_hostname=None): net_inspection_gateway = _make_attr_filter('inspection_gateway') -@jinja2.contextfilter +@jinja2.pass_context def net_neutron_allocation_pool_start(context, name, inventory_hostname=None): return net_attr(context, name, 'neutron_allocation_pool_start', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_neutron_allocation_pool_end(context, name, inventory_hostname=None): return net_attr(context, name, 'neutron_allocation_pool_end', inventory_hostname) @@ -242,12 +242,12 @@ def net_neutron_allocation_pool_end(context, name, inventory_hostname=None): net_neutron_gateway = _make_attr_filter('neutron_gateway') -@jinja2.contextfilter +@jinja2.pass_context def net_vlan(context, name, inventory_hostname=None): return net_attr(context, name, 'vlan', inventory_hostname) -@jinja2.contextfilter +@jinja2.pass_context def net_mtu(context, name, inventory_hostname=None): mtu = net_attr(context, name, 'mtu', inventory_hostname) if mtu is not None: @@ -264,7 +264,7 @@ net_ethtool_opts = _make_attr_filter('ethtool_opts') net_zone = _make_attr_filter('zone') -@jinja2.contextfilter +@jinja2.pass_context def net_libvirt_network_name(context, name, inventory_hostname=None): """Return the configured Libvirt name for a network. @@ -275,7 +275,7 @@ def net_libvirt_network_name(context, name, inventory_hostname=None): return libvirt_name or name -@jinja2.contextfilter +@jinja2.pass_context def net_bridge_ports(context, name, inventory_hostname=None): return net_attr(context, name, 'bridge_ports', inventory_hostname) @@ -326,7 +326,7 @@ def _validate_rules(rules): "for CentOS") -@jinja2.contextfilter +@jinja2.pass_context def net_interface_obj(context, name, inventory_hostname=None): """Return a dict representation of a network interface. @@ -378,7 +378,7 @@ def net_interface_obj(context, name, inventory_hostname=None): return interface -@jinja2.contextfilter +@jinja2.pass_context def net_bridge_obj(context, name, inventory_hostname=None): """Return a dict representation of a network bridge interface. @@ -432,7 +432,7 @@ def net_bridge_obj(context, name, inventory_hostname=None): return interface -@jinja2.contextfilter +@jinja2.pass_context def net_bond_obj(context, name, inventory_hostname=None): """Return a dict representation of a network bond interface. @@ -523,27 +523,27 @@ def _net_interface_type(context, name, inventory_hostname): return 'bond' -@jinja2.contextfilter +@jinja2.pass_context def net_is_ether(context, name, inventory_hostname=None): return _net_interface_type(context, name, inventory_hostname) == 'ether' -@jinja2.contextfilter +@jinja2.pass_context def net_is_bridge(context, name, inventory_hostname=None): return _net_interface_type(context, name, inventory_hostname) == 'bridge' -@jinja2.contextfilter +@jinja2.pass_context def net_is_bond(context, name, inventory_hostname=None): return _net_interface_type(context, name, inventory_hostname) == 'bond' -@jinja2.contextfilter +@jinja2.pass_context def net_is_vlan(context, name, inventory_hostname=None): return net_vlan(context, name) is not None -@jinja2.contextfilter +@jinja2.pass_context def net_is_vlan_interface(context, name, inventory_hostname=None): device = get_and_validate_interface(context, name, inventory_hostname) # Use a heuristic to match conventional VLAN names, ending with a @@ -551,43 +551,43 @@ def net_is_vlan_interface(context, name, inventory_hostname=None): return re.match(r"^[a-zA-Z0-9_\-]+\.[1-9][\d]{0,3}$", device) -@jinja2.contextfilter +@jinja2.pass_context def net_select_ethers(context, names, inventory_hostname=None): return [name for name in names if net_is_ether(context, name, inventory_hostname)] -@jinja2.contextfilter +@jinja2.pass_context def net_select_bridges(context, names, inventory_hostname=None): return [name for name in names if net_is_bridge(context, name, inventory_hostname)] -@jinja2.contextfilter +@jinja2.pass_context def net_select_bonds(context, names, inventory_hostname=None): return [name for name in names if net_is_bond(context, name, inventory_hostname)] -@jinja2.contextfilter +@jinja2.pass_context def net_select_vlans(context, names, inventory_hostname=None): return [name for name in names if net_is_vlan(context, name, inventory_hostname)] -@jinja2.contextfilter +@jinja2.pass_context def net_select_vlan_interfaces(context, names, inventory_hostname=None): return [name for name in names if net_is_vlan_interface(context, name, inventory_hostname)] -@jinja2.contextfilter +@jinja2.pass_context def net_reject_vlans(context, names, inventory_hostname=None): return [name for name in names if not net_is_vlan(context, name, inventory_hostname)] -@jinja2.contextfilter +@jinja2.pass_context def net_configdrive_network_device(context, name, inventory_hostname=None): device = net_interface(context, name, inventory_hostname) if not device: @@ -619,7 +619,7 @@ def net_configdrive_network_device(context, name, inventory_hostname=None): return interface -@jinja2.contextfilter +@jinja2.pass_context def net_libvirt_network(context, name, inventory_hostname=None): """Return a dict which describes the Libvirt network for a network. @@ -634,7 +634,7 @@ def net_libvirt_network(context, name, inventory_hostname=None): } -@jinja2.contextfilter +@jinja2.pass_context def net_libvirt_vm_network(context, name, inventory_hostname=None): """Return a dict which describes the Libvirt VM's network for a network. @@ -648,7 +648,7 @@ def net_libvirt_vm_network(context, name, inventory_hostname=None): } -@jinja2.contextfilter +@jinja2.pass_context def net_ovs_veths(context, names, inventory_hostname=None): """Return a list of virtual Ethernet pairs for OVS. diff --git a/kayobe/tests/unit/plugins/action/test_kolla_ansible_host_vars.py b/kayobe/tests/unit/plugins/action/test_kolla_ansible_host_vars.py index 43ec63d86..7be386d5a 100644 --- a/kayobe/tests/unit/plugins/action/test_kolla_ansible_host_vars.py +++ b/kayobe/tests/unit/plugins/action/test_kolla_ansible_host_vars.py @@ -20,17 +20,17 @@ import jinja2 from kayobe.plugins.action import kolla_ansible_host_vars -@jinja2.contextfilter +@jinja2.pass_context def _net_interface(context, name): return context.get(name + '_interface') -@jinja2.contextfilter +@jinja2.pass_context def _net_vlan(context, name): return context.get(name + '_vlan') -@jinja2.contextfilter +@jinja2.pass_context def _net_select_bridges(context, names): return [name for name in names if (_net_interface(context, name) or "").startswith("br")] diff --git a/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 b/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 new file mode 100644 index 000000000..b429c01c1 --- /dev/null +++ b/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 @@ -0,0 +1,7 @@ +--- +# Don't build an IPA deployment image. +create_ipa_image: false +download_ipa: false + +# Don't build a disk image. It takes time and can be unreliable. +create_image_via_dib: false diff --git a/playbooks/kayobe-seed-base/overrides.yml.j2 b/playbooks/kayobe-seed-base/overrides.yml.j2 index df0b53645..5fe3c4279 100644 --- a/playbooks/kayobe-seed-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-base/overrides.yml.j2 @@ -32,11 +32,3 @@ pip_trusted_hosts: # are using for SSH to be removed. Use a dummy interface. aio_bridge_ports: - dummy1 - -# Build seed deployment images (IPA) with extra-hardware element -ipa_build_images: true -ipa_build_dib_elements_extra: - - "extra-hardware" - -# Build overcloud host image -overcloud_dib_build_host_images: true diff --git a/playbooks/kayobe-seed-base/pre.yml b/playbooks/kayobe-seed-base/pre.yml index 1d4bc5e9a..7db70d533 100644 --- a/playbooks/kayobe-seed-base/pre.yml +++ b/playbooks/kayobe-seed-base/pre.yml @@ -30,6 +30,12 @@ path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost" state: "directory" + # NOTE(mgoddard): Use dib.yml, since it takes precedence over bifrost.yml. + - name: Ensure bifrost overrides file exists + template: + src: bifrost-overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/dib.yml" + - name: Ensure kayobe is installed shell: cmd: dev/install.sh &> {{ logs_dir }}/ansible/install diff --git a/releasenotes/notes/jinja2-pass-context-fecf00f23e413393.yaml b/releasenotes/notes/jinja2-pass-context-fecf00f23e413393.yaml new file mode 100644 index 000000000..3a7ecc729 --- /dev/null +++ b/releasenotes/notes/jinja2-pass-context-fecf00f23e413393.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes an issue seen when using Jinja2 3.1.0. diff --git a/requirements.txt b/requirements.txt index 634aff0e1..bab4725ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pbr>=2.0 # Apache-2.0 +Jinja2>3 # BSD ansible>=4,<6.0 # GPLv3 cliff>=3.1.0 # Apache netaddr!=0.7.16,>=0.7.13 # BSD diff --git a/requirements.yml b/requirements.yml index 4851b821b..bd8629ebd 100644 --- a/requirements.yml +++ b/requirements.yml @@ -36,7 +36,7 @@ roles: - src: stackhpc.libvirt-vm version: v1.14.2 - src: stackhpc.luks - version: 0.4.1 + version: 0.4.2 - src: stackhpc.mellanox-switch version: v1.0.0 - src: stackhpc.os-images