Merge "Make setup module arguments configurable"

ansible/group_vars/all/globals

@@ -51,3 +51,14 @@ os_distribution: "centos"
 # OS release. Valid options are "8-stream" when os_distribution is "centos", or
 # "focal" when os_distribution is "ubuntu".
 os_release: "{{ '8-stream' if os_distribution == 'centos' else 'focal' }}"
+
+###############################################################################
+# Ansible configuration.
+
+# Filter to apply to the setup module when gathering facts. Default is to not
+# specify a filter.
+kayobe_ansible_setup_filter: "{{ omit }}"
+
+# Gather subset to apply to the setup module when gathering facts. Default is
+# to not specify a gather subset.
+kayobe_ansible_setup_gather_subset: "{{ omit }}"

ansible/kayobe-target-venv.yml

@@ -19,7 +19,9 @@
     - block:
         - name: Gather facts
           setup:
-          when: not ansible_facts.module_setup | default(false)
+            filter: "{{ kayobe_ansible_setup_filter }}"
+            gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
+          when: not ansible_facts
           register: gather_facts
 
         - name: Ensure the Python virtualenv package is installed
@@ -79,6 +81,8 @@
     # again using the interpreter from the virtual environment.
     - name: Gather facts
       setup:
+        filter: "{{ kayobe_ansible_setup_filter }}"
+        gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
       when:
         - virtualenv is defined
         - gather_facts is not skipped

ansible/kolla-target-venv.yml

@@ -21,7 +21,9 @@
     - block:
         - name: Gather facts
           setup:
-          when: not ansible_facts.module_setup | default(false)
+            filter: "{{ kayobe_ansible_setup_filter }}"
+            gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
+          when: not ansible_facts
 
         - name: Ensure the Python virtualenv package is installed
           package:

ansible/overcloud-facts-gather.yml

@@ -0,0 +1,10 @@
+---
+- name: Gather facts
+  hosts: overcloud
+  gather_facts: false
+  tasks:
+    - name: Gather facts
+      setup:
+        filter: "{{ kayobe_ansible_setup_filter }}"
+        gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
+      when: not ansible_facts

ansible/roles/network-redhat/tasks/main.yml

@@ -36,6 +36,8 @@
          net_select_bonds |
          map('net_bond_obj') |
          list }}
+    interfaces_setup_filter: "{{ kayobe_ansible_setup_filter }}"
+    interfaces_setup_gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
 
 # Configure virtual ethernet patch links to connect the workload provision
 # and external network bridges to the Neutron OVS bridge.

doc/source/administration/overcloud.rst

@@ -249,3 +249,15 @@ Or to perform an incremental backup, run the following command:
 Further information on backing up and restoring the database is available in
 the :kolla-ansible-doc:`Kolla Ansible documentation
 <admin/mariadb-backup-and-restore.html>`.
+
+Gathering Facts
+===============
+
+The following command may be used to gather facts for all overcloud hosts, for
+both Kayobe and Kolla Ansible:
+
+.. code-block:: console
+
+   kayobe overcloud facts gather
+
+This may be useful to populate a fact cache in advance of other operations.

doc/source/configuration/reference/ansible.rst

@@ -84,3 +84,65 @@ caching using the `jsonfile cache plugin
 
 You may also wish to set the expiration timeout for the cache via ``[defaults]
 fact_caching_timeout``.
+
+Fact gathering
+==============
+
+Fact filtering
+--------------
+
+Filtering of facts can be used to speed up Ansible.  Environments with
+many network interfaces on the network and compute nodes can experience very
+slow processing with Kayobe and Kolla Ansible. This happens due to the
+processing of the large per-interface facts with each task.  To avoid storing
+certain facts, we can use the ``kayobe_ansible_setup_filter`` variable, which
+is used as the ``filter`` argument to the ``setup`` module.
+
+One case where this is particularly useful is to avoid collecting facts for
+virtual tap (beginning with t) and bridge (beginning with q) interfaces
+created by Neutron. These facts are large map values which can consume a lot
+of resources on the Ansible control host. Kayobe and Kolla Ansible typically
+do not need to reference them, so they may be filtered. For example, to
+avoid collecting facts beginning with q or t:
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/globals.yml``
+
+   kayobe_ansible_setup_filter: "ansible_[!qt]*"
+
+Similarly, for Kolla Ansible (notice the similar but different file names):
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/kolla/globals.yml``
+
+   kolla_ansible_setup_filter: "ansible_[!qt]*"
+
+This causes Ansible to collect but not store facts matching that pattern, which
+includes the virtual interface facts. Currently we are not referencing other
+facts matching the pattern within Kolla Ansible.  Note that including the
+'ansible_' prefix causes meta facts ``module_setup`` and ``gather_subset`` to
+be filtered, but this seems to be the only way to get a good match on the
+interface facts.
+
+The exact improvement will vary, but has been reported to be as large as 18x on
+systems with many virtual interfaces.
+
+Fact gathering subsets
+----------------------
+
+It is also possible to configure which subsets of facts are gathered, via
+``kayobe_ansible_setup_gather_subset``, which is used as the ``gather_subset``
+argument to the ``setup`` module. For example, if one wants to avoid collecting
+facts via facter:
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/globals.yml``
+
+   kayobe_ansible_setup_gather_subset: "all,!facter"
+
+Similarly, for Kolla Ansible (notice the similar but different file names):
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/kolla/globals.yml``
+
+   kolla_ansible_setup_gather_subset: "all,!facter"

etc/kayobe/globals.yml

@@ -53,6 +53,17 @@
 # "focal" when os_distribution is "ubuntu".
 #os_release:
 
+###############################################################################
+# Ansible configuration.
+
+# Filter to apply to the setup module when gathering facts. Default is to not
+# specify a filter.
+#kayobe_ansible_setup_filter:
+
+# Gather subset to apply to the setup module when gathering facts. Default is
+# to not specify a gather subset.
+#kayobe_ansible_setup_gather_subset:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

kayobe/cli/commands.py

@@ -932,6 +932,22 @@ class OvercloudDeprovision(KayobeAnsibleMixin, VaultMixin, Command):
         self.run_kayobe_playbooks(parsed_args, playbooks)
 
 
+class OvercloudFactsGather(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
+                           Command):
+    """Gather facts for Kayobe and Kolla Ansible."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Gathering overcloud host facts")
+
+        # Gather facts for Kayobe.
+        playbooks = _build_playbook_list("overcloud-facts-gather")
+        self.run_kayobe_playbooks(parsed_args, playbooks)
+
+        # Gather facts for Kolla Ansible.
+        self.generate_kolla_ansible_config(parsed_args, service_config=False)
+        self.run_kolla_ansible_overcloud(parsed_args, "gather-facts")
+
+
 class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
                              Command):
     """Configure the overcloud host OS and services.

kayobe/tests/unit/cli/test_commands.py

@@ -1008,6 +1008,43 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
 
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_facts_gather(self, mock_kolla_run, mock_run):
+        command = commands.OvercloudFactsGather(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path(
+                        "ansible", "overcloud-facts-gather.yml"),
+                ],
+            ),
+            mock.call(
+                mock.ANY,
+                [utils.get_data_files_path("ansible", "kolla-ansible.yml")],
+                tags="config",
+                ignore_limit=True,
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "gather-facts"
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     @mock.patch.object(commands.KollaAnsibleMixin,

releasenotes/notes/setup-module-args-2c36e56bf78ab5f0.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Adds support for configuring the ``filter`` and ``gather_subset`` arguments
+    for the ``setup`` module via ``kayobe_ansible_setup_filter`` and
+    ``kayobe_ansible_setup_gather_subset`` respectively. These can be used to
+    reduce the number of facts, which can have a significant effect on
+    performance of Ansible.
+  - |
+    Adds a new command, ``kayobe overcloud facts gather``, to gather Ansible
+    facts for overcloud hosts. This may be useful for populating a fact cache.

setup.cfg

@@ -58,6 +58,7 @@ kayobe.cli=
     overcloud_database_recover = kayobe.cli.commands:OvercloudDatabaseRecover
     overcloud_deployment_image_build = kayobe.cli.commands:OvercloudDeploymentImageBuild
     overcloud_deprovision = kayobe.cli.commands:OvercloudDeprovision
+    overcloud_facts_gather = kayobe.cli.commands:OvercloudFactsGather
     overcloud_hardware_inspect = kayobe.cli.commands:OvercloudHardwareInspect
     overcloud_host_configure = kayobe.cli.commands:OvercloudHostConfigure
     overcloud_host_package_update = kayobe.cli.commands:OvercloudHostPackageUpdate
@@ -132,6 +133,8 @@ kayobe.cli.overcloud_deployment_image_build =
     hooks = kayobe.cli.commands:HookDispatcher
 kayobe.cli.overcloud_deprovision =
     hooks = kayobe.cli.commands:HookDispatcher
+kayobe.cli.overcloud_facts_gather =
+    hooks = kayobe.cli.commands:HookDispatcher
 kayobe.cli.overcloud_hardware_inspect =
     hooks = kayobe.cli.commands:HookDispatcher
 kayobe.cli.overcloud_host_configure =