Handle projects still using pbr autodoc_index_modules

There are a non-zero number of projects in OpenStack that are still
using pbr.autodoc_index_modules. Switching to the new sphinx build job
breaks them if they have warning-is-error turned on, as their docs are
expecting a reference to the autogenerated module doc indexes.

We should come up with a better solution for migrating them ... probably
a sphinx plugin that one can use with some options and whatnot. Or we
can commit a copy of the generated autoindex file and then treat it as
actual source code moving forward.

For now we need to run those projects with the pbr modified sphinx
builder. This adds support to the setup.cfg value lookup module
for looking up autodoc_index_modules and plumbs it in to the sphinx
invocation.

A note has also been left with an idea for how to simplify this and
remove the python module (we can ini lookup on the executor) but
implementing that is left as an exercise for later.

Change-Id: Ie0c9f24df09255e871f904e079b68809144b36b4

roles/sphinx/library/sphinx_check_warning_is_error.py

@@ -47,6 +47,19 @@ import os
 from ansible.module_utils.basic import AnsibleModule
 
 
+# TODO(mordred) We should replace this python module with the 'ini' lookup
+# module. That runs on the executor, but the executor has a readable copy of
+# the setup.cfg for the project in question.
+# Just a simple:
+#   set_fact:
+#     setup_cfg_path: "{{ zuul.executor.work_root }}/"\
+#                     "{{ zuul.project.src_dir }}/setup.cfg }}"
+#   warning_is_error: "{{ lookup('ini', 'warning[_\_]is[_\-]error section="\
+#                     "build_sphinx re=True file=' + setup_cfg_path }}"
+#   autodoc_index_modules: "{{ lookup('ini', 'autodoc[_\_]index[_\-]modules"\
+#                          "section=build_sphinx re=True file=' + "\
+#                          "setup_cfg_path }}"
+# should do it. That's a bigger change that we should test aggressively.
 def main():
     module = AnsibleModule(
         argument_spec=dict(
@@ -55,25 +68,38 @@ def main():
     )
     project_dir = module.params['project_dir']
 
+    warning_is_error = False
+    # TODO(mordred) Remove autodoc_index_modules logic  when we get OpenStack
+    # projects off of the pbr autoindex
+    autodoc_index_modules = False
+
     if not os.path.exists(os.path.join(project_dir, 'setup.cfg')):
         module.exit_json(
             changed=False,
-            warning_is_error=False,
+            warning_is_error=warning_is_error,
+            autodoc_index_modules=autodoc_index_modules,
             msg="No setup.cfg, no action needed")
 
     try:
         c = configparser.ConfigParser()
         c.read(os.path.join(project_dir, 'setup.cfg'))
-        warning_is_error = c.getboolean('build_sphinx', 'warning-is-error')
     except Exception:
         module.exit_json(
             changed=False,
-            warning_is_error=False,
-            msg="Setting not found in setup.cfg, defaulting to false")
+            warning_is_error=warning_is_error,
+            autodoc_index_modules=autodoc_index_modules,
+            msg="Error reading setup.cfg, defaulting flags to false")
+
+    if (c.has_section('build_sphinx') and
+            c.has_option('build_sphinx', 'warning-is-error')):
+        warning_is_error = c.getboolean('build_sphinx', 'warning-is-error')
+    if c.has_section('pbr') and c.has_option('pbr', 'autodoc_index_modules'):
+        autodoc_index_modules = c.getboolean('pbr', 'autodoc_index_modules')
     module.exit_json(
         changed=False,
         warning_is_error=warning_is_error,
-        msg="warning_is_error setting found in build_sphinx section")
+        autodoc_index_modules=autodoc_index_modules,
+        msg="Doc building options found in setup.cfg")
 
 
 if __name__ == '__main__':

roles/sphinx/tasks/main.yaml

@@ -8,6 +8,7 @@
   when: sphinx_warning_is_error is not defined
   set_fact:
     sphinx_warning_is_error: "{{ check_result.warning_is_error }}"
+    sphinx_pbr_autodoc: "{{ check_result.autodoc_index_modules }}"
 
 - name: Run sphinx
   shell:
@@ -21,6 +22,19 @@
         {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }}
     chdir: "{{ zuul_work_dir }}"
   with_items: "{{ sphinx_builders }}"
+  when: not sphinx_pbr_autodoc
+
+# TODO(mordred) Remove this when we get projects off of the pbr autoindex
+- name: Run sphinx using legacy pbr runner
+  shell:
+    executable: /bin/bash
+    cmd: |
+      # Source the activate file so that sphinx subcommands have the correct
+      # paths set.
+      source {{ zuul_work_virtualenv }}/bin/activate
+      python setup.py build_sphinx
+    chdir: "{{ zuul_work_dir }}"
+  when: sphinx_pbr_autodoc
 
 - name: Check for whereto
   stat: