From 56ee80234cae7336d2857f88b6f130a499c0c8eb Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Thu, 12 Sep 2019 17:58:48 +0000
Subject: [PATCH] fetch-tox-output: introduce zuul_use_fetch_output
This change enables using fetch-tox role along with
the fetch-output role. By default the role still synchronizes
artifacts back to the executor.
Change-Id: Iffe5ffc52ee4c765b2e23ab8b3235b6bc7c510d8
---
roles/fetch-tox-output/README.rst | 8 +++
roles/fetch-tox-output/defaults/main.yaml | 3 +
roles/fetch-tox-output/tasks/main.yaml | 18 ++++++
test-playbooks/python/fetch-tox-output.yaml | 67 +++++++++++++++++++++
zuul-tests.d/python-jobs.yaml | 22 +++++++
5 files changed, 118 insertions(+)
create mode 100644 test-playbooks/python/fetch-tox-output.yaml
diff --git a/roles/fetch-tox-output/README.rst b/roles/fetch-tox-output/README.rst
index 7fe6f1219..a20fb1f4d 100644
--- a/roles/fetch-tox-output/README.rst
+++ b/roles/fetch-tox-output/README.rst
@@ -16,3 +16,11 @@ Collect log output from a tox build
:default: {{ zuul.project.src_dir }}
Directory tox was run in.
+
+.. zuul:rolevar:: zuul_use_fetch_output
+ :default: false
+
+ Whether to synchronize files to the executor work dir, or to copy them
+ on the test instance.
+ When set to false, the role synchronizes the file to the executor.
+ When set to true, the job needs to use the fetch-output role later.
diff --git a/roles/fetch-tox-output/defaults/main.yaml b/roles/fetch-tox-output/defaults/main.yaml
index dddc1c877..cce0be99e 100644
--- a/roles/fetch-tox-output/defaults/main.yaml
+++ b/roles/fetch-tox-output/defaults/main.yaml
@@ -5,3 +5,6 @@ tox_envlist: venv
tox_executable: tox
zuul_work_dir: "{{ zuul.project.src_dir }}"
+
+zuul_output_dir: "{{ ansible_user_dir }}/zuul-output"
+zuul_use_fetch_output: "{{ zuul_site_use_fetch_output|default(false) }}"
diff --git a/roles/fetch-tox-output/tasks/main.yaml b/roles/fetch-tox-output/tasks/main.yaml
index 2fe05359e..c6540cd16 100644
--- a/roles/fetch-tox-output/tasks/main.yaml
+++ b/roles/fetch-tox-output/tasks/main.yaml
@@ -14,6 +14,12 @@
state: directory
delegate_to: localhost
+- name: Ensure zuul-output tox dir
+ file:
+ path: "{{ zuul_output_dir }}/logs/tox"
+ state: directory
+ when: zuul_use_fetch_output
+
- name: Set envlist fact
set_fact:
envlist: "{{ tox_envlist.split(',') }}"
@@ -31,6 +37,17 @@
envlist: "{{ tox_environments.stdout_lines }}"
when: tox_envlist is not defined or tox_envlist|lower == 'all'
+- name: Copy tox logs
+ copy:
+ dest: "{{ zuul_output_dir }}/logs/tox/"
+ src: "{{ zuul_work_dir }}/.tox/{{ item }}/log/"
+ remote_src: true
+ with_items: "{{ envlist }}"
+ # some tox runs may not create a virtualenv and thus have
+ # no ./tox/env directory
+ failed_when: false
+ when: zuul_use_fetch_output
+
- name: Collect tox logs
synchronize:
dest: "{{ log_path }}"
@@ -41,3 +58,4 @@
# some tox runs may not create a virtualenv and thus have
# no ./tox/env directory
failed_when: false
+ when: not zuul_use_fetch_output
diff --git a/test-playbooks/python/fetch-tox-output.yaml b/test-playbooks/python/fetch-tox-output.yaml
new file mode 100644
index 000000000..9812fb185
--- /dev/null
+++ b/test-playbooks/python/fetch-tox-output.yaml
@@ -0,0 +1,67 @@
+- hosts: all
+ pre_tasks:
+ # Run ensure-output-dirs now as it is not performed speculatively
+ - import_role:
+ name: ensure-output-dirs
+
+ - name: Simplify tox config
+ copy:
+ content: "{{ item.content }}"
+ dest: "{{ zuul.project.src_dir }}/{{ item.dest }}"
+ with_items:
+ - content: |
+ [testenv]
+ sitepackages = True
+ usedevelop = True
+
+ [testenv:venv]
+ commands = python -c 'exit(0)'
+ dest: tox.ini
+ - content: |
+ import setuptools
+ setuptools.setup()
+ dest: setup.py
+ - content: ""
+ dest: setup.cfg
+
+ - name: Generate tox results
+ include_role:
+ name: "{{ item }}"
+ with_items:
+ - ensure-tox
+ - ensure-python
+ - tox
+
+ tasks:
+ - import_role:
+ name: fetch-tox-output
+
+ - block:
+ - name: Undo the log_path fact set by fetch-javascript-output
+ set_fact:
+ log_path: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+ when: groups['all'] | length > 1
+
+ - name: Undo the log_path fact set log path for single node
+ set_fact:
+ log_path: "{{ zuul.executor.log_root }}"
+ when: groups['all'] | length == 1
+
+ - import_role:
+ name: fetch-output
+ when: zuul_use_fetch_output
+
+ post_tasks:
+ - name: Check for artifact on the test instance
+ stat:
+ path: "{{ ansible_user_dir }}/zuul-output/logs/tox/venv-0.log"
+ register: _test_artifact
+ failed_when: not _test_artifact.stat.exists
+ when: zuul_use_fetch_output
+
+ - name: Check for artifact on the executor
+ stat:
+ path: "{{ zuul.executor.log_root }}/tox/venv-0.log"
+ delegate_to: localhost
+ register: _executor_artifact
+ failed_when: not _executor_artifact.stat.exists
diff --git a/zuul-tests.d/python-jobs.yaml b/zuul-tests.d/python-jobs.yaml
index 79f987c26..e2dafeb0b 100644
--- a/zuul-tests.d/python-jobs.yaml
+++ b/zuul-tests.d/python-jobs.yaml
@@ -22,10 +22,32 @@
# tools/update-test-platforms.py
# Please re-run to generate new job lists
+- job:
+ name: zuul-jobs-test-fetch-tox-output
+ description: Test the fetch-tox-output
+ files:
+ - roles/ensure-output-dirs/.*
+ - roles/fetch-tox-output/.*
+ - roles/fetch-output/.*
+ run: test-playbooks/python/fetch-tox-output.yaml
+ vars:
+ zuul_use_fetch_output: true
+
+- job:
+ name: zuul-jobs-test-fetch-tox-output-synchronize
+ description: Test the fetch-tox-output
+ files:
+ - roles/fetch-tox-output/.*
+ run: test-playbooks/python/fetch-tox-output.yaml
+ vars:
+ zuul_use_fetch_output: false
+
- project:
check:
jobs: &id001
- zuul-jobs-test-ensure-sphinx
- zuul-jobs-test-tox-siblings
+ - zuul-jobs-test-fetch-tox-output
+ - zuul-jobs-test-fetch-tox-output-synchronize
gate:
jobs: *id001