From 50fd13464603586b4babace781288eaef14c93af Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Fri, 16 Dec 2022 12:32:16 -0800
Subject: [PATCH] Add nox role and some simple jobs
This is an alternative to tox.
Change-Id: Ib4920acec09c2c980af909e8f9d1eabd1c6d253a
---
doc/source/python-jobs.rst | 11 +
doc/source/python-roles.rst | 1 +
playbooks/nox/cover-post.yaml | 3 +
playbooks/nox/docs-post.yaml | 3 +
playbooks/nox/docs-pre.yaml | 4 +
playbooks/nox/pre.yaml | 4 +
playbooks/nox/run.yaml | 4 +
playbooks/unittests/pre.yaml | 3 +-
roles/nox/README.rst | 84 +++++
roles/nox/__init__.py | 0
roles/nox/defaults/main.yaml | 10 +
roles/nox/library/__init__.py | 0
.../library/nox_install_sibling_packages.py | 347 ++++++++++++++++++
roles/nox/library/test-constraints.txt | 2 +
.../test_nox_install_sibling_packages.py | 61 +++
roles/nox/tasks/main.yaml | 83 +++++
roles/nox/tasks/siblings.yaml | 56 +++
zuul.d/python-jobs.yaml | 185 ++++++++++
18 files changed, 860 insertions(+), 1 deletion(-)
create mode 100644 playbooks/nox/cover-post.yaml
create mode 100644 playbooks/nox/docs-post.yaml
create mode 100644 playbooks/nox/docs-pre.yaml
create mode 100644 playbooks/nox/pre.yaml
create mode 100644 playbooks/nox/run.yaml
create mode 100644 roles/nox/README.rst
create mode 100644 roles/nox/__init__.py
create mode 100644 roles/nox/defaults/main.yaml
create mode 100644 roles/nox/library/__init__.py
create mode 100644 roles/nox/library/nox_install_sibling_packages.py
create mode 100644 roles/nox/library/test-constraints.txt
create mode 100644 roles/nox/library/test_nox_install_sibling_packages.py
create mode 100644 roles/nox/tasks/main.yaml
create mode 100644 roles/nox/tasks/siblings.yaml
diff --git a/doc/source/python-jobs.rst b/doc/source/python-jobs.rst
index 363b2c1f1..67f67837d 100644
--- a/doc/source/python-jobs.rst
+++ b/doc/source/python-jobs.rst
@@ -17,6 +17,17 @@ Python Jobs
.. zuul:autojob:: tox-cover
.. zuul:autojob:: tox-bashate
.. zuul:autojob:: tox-nodejs-npm
+.. zuul:autojob:: nox
+.. zuul:autojob:: nox-py27
+.. zuul:autojob:: nox-py36
+.. zuul:autojob:: nox-py37
+.. zuul:autojob:: nox-py38
+.. zuul:autojob:: nox-py39
+.. zuul:autojob:: nox-py310
+.. zuul:autojob:: nox-py311
+.. zuul:autojob:: nox-docs
+.. zuul:autojob:: nox-linters
+.. zuul:autojob:: nox-cover
.. zuul:autojob:: build-python-release
.. zuul:autojob:: python-upload-pypi
.. zuul:autojob:: build-sphinx-docs
diff --git a/doc/source/python-roles.rst b/doc/source/python-roles.rst
index 7a6b87626..b99790396 100644
--- a/doc/source/python-roles.rst
+++ b/doc/source/python-roles.rst
@@ -21,4 +21,5 @@ Python Roles
.. zuul:autorole:: find-constraints
.. zuul:autorole:: sphinx
.. zuul:autorole:: tox
+.. zuul:autorole:: nox
.. zuul:autorole:: upload-pypi
diff --git a/playbooks/nox/cover-post.yaml b/playbooks/nox/cover-post.yaml
new file mode 100644
index 000000000..946b0c101
--- /dev/null
+++ b/playbooks/nox/cover-post.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+ roles:
+ - role: fetch-coverage-output
diff --git a/playbooks/nox/docs-post.yaml b/playbooks/nox/docs-post.yaml
new file mode 100644
index 000000000..b6f4ecafe
--- /dev/null
+++ b/playbooks/nox/docs-post.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+ roles:
+ - fetch-sphinx-tarball
diff --git a/playbooks/nox/docs-pre.yaml b/playbooks/nox/docs-pre.yaml
new file mode 100644
index 000000000..693ef41e3
--- /dev/null
+++ b/playbooks/nox/docs-pre.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+ roles:
+ - role: bindep
+ bindep_dir: "{{ zuul_work_dir }}"
diff --git a/playbooks/nox/pre.yaml b/playbooks/nox/pre.yaml
new file mode 100644
index 000000000..f3221e719
--- /dev/null
+++ b/playbooks/nox/pre.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+ roles:
+ - ensure-python
+ - ensure-nox
diff --git a/playbooks/nox/run.yaml b/playbooks/nox/run.yaml
new file mode 100644
index 000000000..7f4913bbd
--- /dev/null
+++ b/playbooks/nox/run.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+ roles:
+ - revoke-sudo
+ - nox
diff --git a/playbooks/unittests/pre.yaml b/playbooks/unittests/pre.yaml
index e046ef386..0afbeef4e 100644
--- a/playbooks/unittests/pre.yaml
+++ b/playbooks/unittests/pre.yaml
@@ -5,7 +5,8 @@
name: bindep
vars:
bindep_dir: "{{ zuul_work_dir }}"
- when: tox_install_bindep | default(true)
+ # TODO don't make this runtime specific
+ when: (tox_install_bindep | default(true)) and (nox_install_bindep | default(true))
- name: Run test-setup role
include_role:
name: test-setup
diff --git a/roles/nox/README.rst b/roles/nox/README.rst
new file mode 100644
index 000000000..782389297
--- /dev/null
+++ b/roles/nox/README.rst
@@ -0,0 +1,84 @@
+Runs nox for a project
+
+This role overrides Python packages installed into nox environments with
+corresponding Zuul sibling projects and runs nox tests as follows:
+
+#. Create nox environments. Note this role currently relies on using
+ the default .nox/session name environment paths.
+#. Get Python sibling package names for sibling projects created by
+ Zuul (using ``required-projects`` job variable). Package names are
+ searched in following sources:
+
+ * ``setup.cfg`` of *pbr* projects,
+ * ``setup.py``,
+ * ``nox_package_name`` role variable.
+
+#. Remove sibling packages from nox environments.
+#. Create temporary constraints file, lines for sibling packages are
+ removed.
+#. Install sibling packages from Zuul projects into nox environments
+ with temporary constraints file.
+#. Run nox tests.
+
+**Role Variables**
+
+.. zuul:rolevar:: nox_environment
+ :type: dict
+ :default: { "CI": "1" }
+
+ Environment variables to pass in to the nox run. Nox behaves differently
+ when the CI env var is set. We set that by default but allow you to
+ override it if the CI behaviors are not desireable.
+
+.. zuul:rolevar:: nox_session
+
+ Space separated string listing nox sessions to run.
+
+.. zuul:rolevar:: nox_keyword
+
+ String to select nox sessions via keyword rather than session name.
+
+.. zuul:rolevar:: nox_tag
+
+ String to select nox sessions via tag rather than session name.
+
+.. zuul:rolevar:: nox_force_python
+
+ String to force a specific python version to be used in the session.
+ This allows you to request session `tests` be run against python `3.11`.
+
+.. zuul:rolevar:: nox_executable
+ :default: nox
+
+ Location of the nox executable.
+
+.. zuul:rolevar:: nox_config_file
+
+ Path to a nox configuration file. If not specified the nox will look
+ for noxfile.py by default.
+
+.. zuul:rolevar:: nox_extra_args
+ :default: -v
+
+ String of extra command line options to pass to nox.
+
+.. zuul:rolevar:: nox_constraints_file
+
+ Path to a pip constraints file. Will be provided to nox via
+ ``NOX_CONSTRAINTS_FILE``.
+
+.. zuul:rolevar:: nox_install_siblings
+ :default: true
+
+ Flag controlling whether to attempt to install python packages from any
+ other source code repos zuul has checked out. Defaults to True.
+
+.. zuul:rolevar:: nox_package_name
+
+ Allows a user to setup the package name to be used by nox, over reading
+ a setup.cfg file in the project.
+
+.. zuul:rolevar:: zuul_work_dir
+ :default: {{ zuul.project.src_dir }}
+
+ Directory to run nox in.
diff --git a/roles/nox/__init__.py b/roles/nox/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/roles/nox/defaults/main.yaml b/roles/nox/defaults/main.yaml
new file mode 100644
index 000000000..a976ea0a0
--- /dev/null
+++ b/roles/nox/defaults/main.yaml
@@ -0,0 +1,10 @@
+---
+nox_environment:
+ # nox will fail with missing session with this flag set.
+ CI: "1"
+nox_executable: nox
+nox_extra_args: '-v'
+nox_install_siblings: true
+nox_inline_comments: true
+
+zuul_work_dir: "{{ zuul.project.src_dir }}"
diff --git a/roles/nox/library/__init__.py b/roles/nox/library/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/roles/nox/library/nox_install_sibling_packages.py b/roles/nox/library/nox_install_sibling_packages.py
new file mode 100644
index 000000000..6e7a3f1a0
--- /dev/null
+++ b/roles/nox/library/nox_install_sibling_packages.py
@@ -0,0 +1,347 @@
+# Copyright (c) 2017 Red Hat
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+# This was adapted from the tox siblings role.
+
+DOCUMENTATION = '''
+---
+module: nox_install_sibling_packages
+short_description: Install packages needed by nox that have local git versions
+author: Monty Taylor (@mordred) and Clark Boylan
+description:
+ - Looks for git repositories that zuul has placed on the system that provide
+ python packages needed by package nox is testing. If if finds any, it will
+ install them into the nox virtualenv so that subsequent runs of nox will
+ use the provided git versions.
+requirements:
+ - "python >= 3.6"
+options:
+ nox_sessions:
+ description:
+ - output of `nox --list` showing which sessions are selected
+ required: true
+ type: path
+ project_dir:
+ description:
+ - The directory in which the project we care about is in.
+ required: true
+ type: str
+ projects:
+ description:
+ - A list of project dicts that zuul knows about
+ required: true
+ type: list
+'''
+
+try:
+ import configparser
+except ImportError:
+ import ConfigParser as configparser
+
+import os
+import os.path
+import subprocess
+import tempfile
+import traceback
+
+from ansible.module_utils.basic import AnsibleModule
+
+log = list()
+
+
+def to_filename(name):
+ """Convert a project or version name to its filename-escaped form
+ Any '-' characters are currently replaced with '_'.
+
+ Implementation vendored from pkg_resources.to_filename in order to avoid
+ adding an extra runtime dependency.
+ """
+ return name.replace('-', '_')
+
+
+def get_sibling_python_packages(projects, nox_python):
+ '''Finds all python packages that zuul has cloned.
+
+ If someone does a require_project: and then runs a nox job, it can be
+ assumed that what they want to do is to test the two together.
+ '''
+ packages = {}
+
+ for project in projects:
+ root = project['src_dir']
+ package_name = None
+ setup_cfg = os.path.join(root, 'setup.cfg')
+ found_python = False
+ if os.path.exists(setup_cfg):
+ found_python = True
+ c = configparser.ConfigParser()
+ c.read(setup_cfg)
+ try:
+ package_name = c.get('metadata', 'name')
+ packages[package_name] = root
+ except Exception:
+ # Some things have a setup.cfg, but don't keep
+ # metadata in it; fall back to setup.py below
+ log.append(
+ "[metadata] name not found in %s, skipping" % setup_cfg)
+ if not package_name and os.path.exists(os.path.join(root, 'setup.py')):
+ found_python = True
+ # It's a python package but doesn't use pbr, so we need to run
+ # python setup.py --name to get setup.py to tell us what the
+ # package name is.
+ package_name = subprocess.check_output(
+ [os.path.abspath(nox_python), 'setup.py', '--name'],
+ cwd=os.path.abspath(root),
+ stderr=subprocess.STDOUT).decode('utf-8')
+ if package_name:
+ package_name = package_name.strip()
+ packages[package_name] = root
+ if found_python and not package_name:
+ log.append(
+ "Could not find package name for {root}".format(
+ root=root))
+ return packages
+
+
+def get_installed_packages(nox_python):
+ # We use the output of pip freeze here as that is pip's stable public
+ # interface.
+ frozen_pkgs = subprocess.check_output(
+ [nox_python, '-m', 'pip', '-qqq', 'freeze'],
+ stderr=subprocess.STDOUT
+ ).decode('utf-8')
+ # Matches strings of the form:
+ # 1. '<package_name>==<version>'
+ # 2. '# Editable Git install with no remote (<package_name>==<version>)'
+ # 3. '<package_name> @ <URI_reference>' # PEP440, PEP508, PEP610
+ # results <package_name>
+ installed_packages = []
+ for x in frozen_pkgs.split('\n'):
+ if '==' in x:
+ installed_packages.append(x[x.find('(') + 1:].split('==')[0])
+ elif '@' in x:
+ installed_packages.append(x.split('@')[0].rstrip(' \t'))
+ return installed_packages
+
+
+def write_new_constraints_file(constraints, packages):
+ with tempfile.NamedTemporaryFile(mode='w', delete=False) \
+ as constraints_file:
+ constraints_lines = open(constraints, 'r').read().split('\n')
+ for line in constraints_lines:
+ package_name = line.split('===')[0]
+ if package_name in packages:
+ continue
+ constraints_file.write(line)
+ constraints_file.write('\n')
+ return constraints_file.name
+
+
+def _get_package_root(name, sibling_packages):
+ '''
+ Returns a package root from the sibling packages dict.
+
+ If name is not found in sibling_packages, tries again using the 'filename'
+ form of the name returned by the setuptools package resource API.
+
+ :param name: package name
+ :param sibling_packages: dict of python packages that zuul has cloned
+ :returns: the package root (str)
+ :raises: KeyError
+ '''
+ try:
+ pkg_root = sibling_packages[name]
+ except KeyError:
+ pkg_root = sibling_packages[to_filename(name)]
+
+ return pkg_root
+
+
+def find_installed_siblings(nox_python, package_name, sibling_python_packages):
+ installed_sibling_packages = []
+ for dep_name in get_installed_packages(nox_python):
+ log.append(
+ "Found {name} python package installed".format(
+ name=dep_name))
+ if (dep_name == package_name or
+ to_filename(dep_name) == package_name):
+ # We don't need to re-process ourself.
+ # We've filtered ourselves from the source dir list,
+ # but let's be sure nothing is weird.
+ log.append(
+ "Skipping {name} because it's us".format(
+ name=dep_name))
+ continue
+ if dep_name in sibling_python_packages:
+ log.append(
+ "Package {name} on system in {root}".format(
+ name=dep_name,
+ root=sibling_python_packages[dep_name]))
+ installed_sibling_packages.append(dep_name)
+ elif to_filename(dep_name) in sibling_python_packages:
+ real_name = to_filename(dep_name)
+ log.append(
+ "Package {name} ({pkg_name}) on system in {root}".format(
+ name=dep_name,
+ pkg_name=real_name,
+ root=sibling_python_packages[real_name]))
+ # need to use dep_name here for later constraint file rewrite
+ installed_sibling_packages.append(dep_name)
+ return installed_sibling_packages
+
+
+def install_siblings(envdir, projects, package_name, constraints):
+ changed = False
+ nox_python = '{envdir}/bin/python'.format(envdir=envdir)
+
+ sibling_python_packages = get_sibling_python_packages(
+ projects, nox_python)
+ for name, root in sibling_python_packages.items():
+ log.append("Sibling {name} at {root}".format(name=name,
+ root=root))
+
+ installed_sibling_packages = find_installed_siblings(
+ nox_python,
+ package_name,
+ sibling_python_packages)
+
+ if constraints:
+ constraints_file = write_new_constraints_file(
+ constraints, installed_sibling_packages)
+
+ for sibling_package in installed_sibling_packages:
+ changed = True
+ log.append("Uninstalling {name}".format(name=sibling_package))
+ uninstall_output = subprocess.check_output(
+ [nox_python, '-m',
+ 'pip', 'uninstall', '-y', sibling_package],
+ stderr=subprocess.STDOUT)
+ log.extend(uninstall_output.decode('utf-8').split('\n'))
+
+ args = [nox_python, '-m', 'pip', 'install']
+ if constraints:
+ args.extend(['-c', constraints_file])
+
+ pkg_root = _get_package_root(sibling_package,
+ sibling_python_packages)
+ log.append(
+ "Installing {name} from {root} for deps".format(
+ name=sibling_package,
+ root=pkg_root))
+ args.append(pkg_root)
+
+ install_output = subprocess.check_output(args)
+ log.extend(install_output.decode('utf-8').split('\n'))
+
+ for sibling_package in installed_sibling_packages:
+ changed = True
+ pkg_root = _get_package_root(sibling_package,
+ sibling_python_packages)
+ log.append(
+ "Installing {name} from {root}".format(
+ name=sibling_package,
+ root=pkg_root))
+
+ install_output = subprocess.check_output(
+ [nox_python, '-m', 'pip', 'install', '--no-deps',
+ pkg_root])
+ log.extend(install_output.decode('utf-8').split('\n'))
+ return changed
+
+
+def main():
+ module = AnsibleModule(
+ argument_spec=dict(
+ nox_sessions=dict(required=True, type='str'),
+ nox_constraints_file=dict(type='str'),
+ nox_package_name=dict(type='str'),
+ project_dir=dict(required=True, type='str'),
+ projects=dict(required=True, type='list'),
+ )
+ )
+ constraints = module.params.get('nox_constraints_file')
+ nox_package_name = module.params.get('nox_package_name')
+ project_dir = module.params['project_dir']
+ projects = module.params['projects']
+ nox_sessions = module.params.get('nox_sessions')
+
+ sessions = []
+ for line in nox_sessions.split('\n'):
+ if line.startswith('*'):
+ sessions.append(line[1:].strip())
+
+ if not sessions:
+ module.exit_json(
+ changed=False,
+ msg='No sessions to run, no action needed.')
+
+ log.append('Using sessions: {}'.format(sessions))
+
+ if (not nox_package_name
+ and not os.path.exists(os.path.join(project_dir, 'setup.cfg'))
+ ):
+ module.exit_json(changed=False, msg="No setup.cfg, no action needed")
+ if constraints and not os.path.exists(constraints):
+ module.fail_json(msg="Constraints file {constraints} was not found")
+
+ # Who are we?
+ package_name = nox_package_name
+ if not package_name:
+ try:
+ c = configparser.ConfigParser()
+ c.read(os.path.join(project_dir, 'setup.cfg'))
+ package_name = c.get('metadata', 'name')
+ except Exception:
+ module.exit_json(
+ changed=False, msg="No name in setup.cfg, skipping siblings")
+
+ log.append(
+ "Processing siblings for {name} from {project_dir}".format(
+ name=package_name,
+ project_dir=project_dir))
+
+ changed = False
+ for session in sessions:
+ # Nox replaces dots in the session name with dashes when creating
+ # venvs.
+ envdir = os.path.join(project_dir, '.nox', session.replace('.', '-'))
+ if not os.path.exists(envdir):
+ # Nox doesn't appear to allow us to lookup the env dir that was
+ # created in a previous step. We look for it where we expect it
+ # to be and fail otherwise.
+ module.fail_json(msg="Nox session env not found: {envdir}")
+ try:
+ siblings_changed = install_siblings(envdir,
+ projects,
+ package_name,
+ constraints)
+ changed = changed or siblings_changed
+ except subprocess.CalledProcessError as e:
+ tb = traceback.format_exc()
+ log.append(str(e))
+ log.append(tb)
+ log.append("Output:")
+ log.extend(e.output.decode('utf-8').split('\n'))
+ module.fail_json(msg=str(e), log="\n".join(log))
+ except Exception as e:
+ tb = traceback.format_exc()
+ log.append(str(e))
+ log.append(tb)
+ module.fail_json(msg=str(e), log="\n".join(log))
+ module.exit_json(changed=changed, msg="\n".join(log))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/roles/nox/library/test-constraints.txt b/roles/nox/library/test-constraints.txt
new file mode 100644
index 000000000..10e67060f
--- /dev/null
+++ b/roles/nox/library/test-constraints.txt
@@ -0,0 +1,2 @@
+requests===2.18.4
+doesnotexistonpypi===0.0.1
diff --git a/roles/nox/library/test_nox_install_sibling_packages.py b/roles/nox/library/test_nox_install_sibling_packages.py
new file mode 100644
index 000000000..cd2e1f651
--- /dev/null
+++ b/roles/nox/library/test_nox_install_sibling_packages.py
@@ -0,0 +1,61 @@
+# Copyright (C) 2019 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This was adapted from the tox siblings role.
+
+import os
+import sys
+import testtools
+
+from .nox_install_sibling_packages import get_installed_packages
+from .nox_install_sibling_packages import write_new_constraints_file
+
+
+class TestNoxInstallSiblingPackages(testtools.TestCase):
+ def test_get_installed_packages(self):
+ # NOTE(mnaser): Given that we run our tests inside a venv, we can
+ # leverage the virtual environment we use in
+ # unit tests instead of mocking up everything.
+ pkgs = get_installed_packages(sys.executable)
+
+ # NOTE(mnaser): requests should be installed in this virtualenv
+ # but this might fail later if we stop adding requests
+ # in the unit tests.
+ self.assertIn("requests", pkgs)
+
+ def test_write_new_constraints_file(self):
+ # NOTE(mnaser): Given that we run our tests inside a venv, we can
+ # leverage the virtual environment we use in
+ # unit tests instead of mocking up everything.
+ pkgs = get_installed_packages(sys.executable)
+
+ # NOTE(mnaser): requests should be installed in this virtualenv
+ # but this might fail later if we stop adding requests
+ # in the unit tests.
+ test_constraints = os.path.join(os.path.dirname(__file__),
+ 'test-constraints.txt')
+ constraints = write_new_constraints_file(test_constraints, pkgs)
+
+ def cleanup_constraints_file():
+ if os.path.exists(constraints):
+ os.unlink(constraints)
+ self.addCleanup(cleanup_constraints_file)
+
+ self.assertTrue(os.path.exists(constraints))
+ with open(constraints) as f:
+ s = f.read()
+ self.assertNotIn("requests", s)
+ self.assertIn("doesnotexistonpypi", s)
diff --git a/roles/nox/tasks/main.yaml b/roles/nox/tasks/main.yaml
new file mode 100644
index 000000000..260054827
--- /dev/null
+++ b/roles/nox/tasks/main.yaml
@@ -0,0 +1,83 @@
+- name: Check to see if the constraints file exists
+ stat:
+ path: "{{ nox_constraints_file }}"
+ get_checksum: false
+ get_mime: false
+ get_md5: false
+ register: stat_results
+ when: nox_constraints_file is defined
+
+- name: Fail if constraints file is missing
+ when: nox_constraints_file is defined and not stat_results.stat.exists
+ fail:
+ msg: nox_constraints_file is defined but was not found
+
+- name: Record constraints file location
+ set_fact:
+ nox_constraints_env:
+ NOX_CONSTRAINTS_FILE: "{{ nox_constraints_file }}"
+ when: nox_constraints_file is defined
+
+- name: Install nox siblings
+ include_tasks: siblings.yaml
+ when: nox_install_siblings
+
+- name: Emit nox command
+ debug:
+ msg: >-
+ {{ nox_executable }}
+ {% if nox_config_file is defined and nox_config_file %}
+ -f {{ nox_config_file }}
+ {% endif %}
+ {% if nox_session is defined and nox_session %}
+ -s {{ nox_session }}
+ {% endif %}
+ {% if nox_keyword is defined and nox_keyword %}
+ -k {{ nox_keyword }}
+ {% endif %}
+ {% if nox_tag is defined and nox_tag %}
+ -t {{ nox_tag }}
+ {% endif %}
+ {% if nox_force_python is defined and nox_force_python %}
+ --force-python {{ nox_force_python }}
+ {% endif %}
+ {% if nox_install_siblings %}
+ --reuse-existing-virtualenvs --no-install
+ {% endif %}
+ {{ nox_extra_args }}
+
+- name: Run nox
+ block:
+ - name: Run nox
+ args:
+ chdir: "{{ zuul_work_dir }}"
+ environment: "{{ nox_environment | combine(nox_constraints_env | default({})) }}"
+ command: >-
+ {{ nox_executable }}
+ {% if nox_config_file is defined and nox_config_file %}
+ -f {{ nox_config_file }}
+ {% endif %}
+ {% if nox_session is defined and nox_session %}
+ -s {{ nox_session }}
+ {% endif %}
+ {% if nox_keyword is defined and nox_keyword %}
+ -k {{ nox_keyword }}
+ {% endif %}
+ {% if nox_tag is defined and nox_tag %}
+ -t {{ nox_tag }}
+ {% endif %}
+ {% if nox_force_python is defined and nox_force_python %}
+ --force-python {{ nox_force_python }}
+ {% endif %}
+ {% if nox_install_siblings %}
+ --reuse-existing-virtualenvs --no-install
+ {% endif %}
+ {{ nox_extra_args }}
+ register: nox_output
+
+ # Even though any test environment in nox failed we want to
+ # return file comments produced so always run this.
+ always:
+ - name: TODO
+ debug:
+ msg: TODO
diff --git a/roles/nox/tasks/siblings.yaml b/roles/nox/tasks/siblings.yaml
new file mode 100644
index 000000000..36f22cb4d
--- /dev/null
+++ b/roles/nox/tasks/siblings.yaml
@@ -0,0 +1,56 @@
+# Install sibling with nox so we can replace them later
+- name: Run nox without tests
+ command: >-
+ {{ nox_executable }}
+ {% if nox_config_file is defined and nox_config_file %}
+ -f {{ nox_config_file }}
+ {% endif %}
+ {% if nox_session is defined and nox_session %}
+ -s {{ nox_session }}
+ {% endif %}
+ {% if nox_keyword is defined and nox_keyword %}
+ -k {{ nox_keyword }}
+ {% endif %}
+ {% if nox_tag is defined and nox_tag %}
+ -t {{ nox_tag }}
+ {% endif %}
+ {% if nox_force_python is defined and nox_force_python %}
+ --force-python {{ nox_force_python }}
+ {% endif %}
+ --install-only
+ {{ nox_extra_args }}
+ args:
+ chdir: "{{ zuul_work_dir }}"
+ environment: "{{ nox_environment | combine(nox_constraints_env | default({})) }}"
+
+- name: Get nox session list
+ command: >-
+ {{ nox_executable }}
+ {% if nox_config_file is defined and nox_config_file %}
+ -f {{ nox_config_file }}
+ {% endif %}
+ {% if nox_session is defined and nox_session %}
+ -s {{ nox_session }}
+ {% endif %}
+ {% if nox_keyword is defined and nox_keyword %}
+ -k {{ nox_keyword }}
+ {% endif %}
+ {% if nox_tag is defined and nox_tag %}
+ -t {{ nox_tag }}
+ {% endif %}
+ {% if nox_force_python is defined and nox_force_python %}
+ --force-python {{ nox_force_python }}
+ {% endif %}
+ --list
+ args:
+ chdir: "{{ zuul_work_dir }}"
+ environment: "{{ nox_environment | combine(nox_constraints_env | default({})) }}"
+ register: _nox_session_listing
+
+- name: Install any sibling python packages
+ nox_install_sibling_packages:
+ nox_sessions: "{{ _nox_session_listing.stdout }}"
+ nox_constraints_file: "{{ nox_constraints_file | default(omit) }}"
+ nox_package_name: "{{ nox_package_name | default(omit) }}"
+ project_dir: "{{ zuul_work_dir }}"
+ projects: "{{ zuul.projects.values() | selectattr('required') | list }}"
diff --git a/zuul.d/python-jobs.yaml b/zuul.d/python-jobs.yaml
index 384798bfa..0957c8c4e 100644
--- a/zuul.d/python-jobs.yaml
+++ b/zuul.d/python-jobs.yaml
@@ -335,6 +335,191 @@
vars:
npm_command: build
+- job:
+ name: nox
+ parent: unittests
+ description: |
+ Base job containing setup and teardown for nox-based test jobs.
+
+ This performs basic host and general project setup tasks common
+ to all nox unit test jobs.
+
+ Responds to these variables:
+
+ .. zuul:jobvar:: nox_session
+
+ Use the specified nox sessions
+
+ .. zuul:jobvar:: nox_keyword
+
+ Use the specified nox keyword
+
+ .. zuul:jobvar:: nox_tag
+
+ Use the specified nox tag
+
+ .. zuul:jobvar:: nox_force_python
+
+ Force nox to run the selected sessions under this version of python.
+
+ .. zuul:jobvar:: nox_config_file
+
+ Override the default noxfile.py configuration path.
+
+ .. zuul:jobvar:: nox_environment
+ :type: dict
+
+ Environment variables to pass in to the nox run.
+ Nox behaves differently when CI=1 is set. Consider setting this
+ if you override the role defaults.
+
+ .. zuul:jobvar:: nox_extra_args
+
+ String containing extra arguments to append to the nox command line.
+
+ .. zuul:jobvar:: nox_constraints_file
+
+ Path to a pip constraints file. Will be provided to nox in the
+ NOX_CONSTRAINTS_FILE environment variable if it exists.
+
+ .. zuul:jobvar:: nox_install_siblings
+ :default: true
+
+ Override nox requirements that have corresponding zuul git repos
+ on the node by installing the git versions into the nox virtualenv.
+
+ .. zuul:jobvar:: nox_install_bindep
+ :default: true
+
+ Whether or not to run the binary dependencies detection and
+ installation with bindep.
+ run: playbooks/nox/run.yaml
+ pre-run: playbooks/nox/pre.yaml
+
+- job:
+ name: nox-py27
+ parent: nox
+ description: |
+ Run unit tests for a Python project under cPython version 2.7.
+
+ Uses nox with the ``test`` keyword forcing python 2.7.
+ vars:
+ nox_keyword: tests
+ nox_force_python: "2.7"
+ python_version: "2.7"
+
+- job:
+ name: nox-py36
+ parent: nox
+ description: |
+ Run unit tests for a Python project under cPython version 3.6.
+
+ Uses nox with the ``test`` keyword forcing python 3.6.
+ vars:
+ nox_keyword: tests
+ nox_force_python: "3.6"
+ python_version: "3.6"
+
+- job:
+ name: nox-py37
+ parent: nox
+ description: |
+ Run unit tests for a Python project under cPython version 3.7.
+
+ Uses nox with the ``test`` keyword forcing python 3.7.
+ vars:
+ nox_keyword: tests
+ nox_force_python: "3.7"
+ python_version: "3.7"
+
+- job:
+ name: nox-py38
+ parent: nox
+ description: |
+ Run unit tests for a Python project under cPython version 3.8.
+
+ Uses nox with the ``test`` keyword forcing python 3.8.
+ vars:
+ nox_keyword: tests
+ nox_force_python: "3.8"
+ python_version: "3.8"
+
+- job:
+ name: nox-py39
+ parent: nox
+ description: |
+ Run unit tests for a Python project under cPython version 3.9.
+
+ Uses nox with the ``test`` keyword forcing python 3.9.
+ vars:
+ nox_keyword: tests
+ nox_force_python: "3.9"
+ python_version: "3.9"
+
+- job:
+ name: nox-py310
+ parent: nox
+ description: |
+ Run unit tests for a Python project under cPython version 3.10.
+
+ Uses nox with the ``test`` keyword forcing python 3.10.
+ vars:
+ nox_keyword: tests
+ nox_force_python: "3.10"
+ python_version: "3.10"
+
+- job:
+ name: nox-py311
+ parent: nox
+ description: |
+ Run unit tests for a Python project under cPython version 3.11.
+
+ Uses nox with the ``test`` keyword forcing python 3.11.
+ vars:
+ nox_keyword: tests
+ nox_force_python: "3.11"
+ python_version: "3.11"
+
+- job:
+ name: nox-cover
+ parent: nox
+ description: |
+ Run code coverage tests.
+
+ Uses nox with the ``cover`` keyword.
+ post-run: playbooks/nox/cover-post.yaml
+ vars:
+ nox_keyword: cover
+
+- job:
+ name: nox-linters
+ parent: nox
+ description: |
+ Runs code linting tests.
+
+ Uses nox with the ``linters`` keyword.
+ vars:
+ nox_keyword: linters
+ test_setup_skip: true
+
+- job:
+ name: nox-docs
+ # This is not parented to nox since we do not need
+ # the roles from its parent unittests.
+ description: |
+ Run documentation unit tests.
+
+ Uses nox with the ``docs`` keyword.
+ vars:
+ nox_keyword: docs
+ bindep_profile: compile doc
+ run: playbooks/nox/run.yaml
+ pre-run:
+ - playbooks/nox/docs-pre.yaml
+ - playbooks/nox/pre.yaml
+ post-run:
+ - playbooks/nox/docs-post.yaml
+
- job:
name: build-python-release
description: |