From b62efba8a43105a03eb87669fdfb12138f3c95bb Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 28 Aug 2019 14:37:55 -0700 Subject: [PATCH] Add option for object store friendly log paths For logs stored in object storage services we want to be able to shard into different containers. This improves scalability of the object stores. To do this consistently we prefix all log paths with the first 3 characters of the build uuid. This way we get container names like log_123, log_abc, and so on for a total of 4096 containers. This behavior is off by default because users of filesystems stores without a zuul dashboard may still want periodic/ to be filed into a top level dir for browseability. Change-Id: I8e72a6e587edcbdf89b793cad2d7c96c535601e7 --- roles/emit-job-header/README.rst | 10 +++++ roles/set-zuul-log-path-fact/README.rst | 11 +++++ .../set-zuul-log-path-fact/defaults/main.yaml | 1 + roles/set-zuul-log-path-fact/tasks/main.yaml | 45 ++++++++++++++----- .../base-roles/emit-job-header.yaml | 28 +++++++++++- 5 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 roles/set-zuul-log-path-fact/defaults/main.yaml diff --git a/roles/emit-job-header/README.rst b/roles/emit-job-header/README.rst index bdb97c7ed..b67d66b62 100644 --- a/roles/emit-job-header/README.rst +++ b/roles/emit-job-header/README.rst @@ -5,3 +5,13 @@ Log a few lines about the job. .. zuul:rolevar:: zuul_log_url Base URL where logs are to be found. + +.. zuul:rolevar:: zuul_log_path_shard_build + :default: False + + This var is consumed by set-zuul-log-path-fact which emit-job-header + calls into. If you set this you will get log paths prefixed with the + first three characters of the build uuid. This will improve log file + sharding. + + More details can be found at :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build` diff --git a/roles/set-zuul-log-path-fact/README.rst b/roles/set-zuul-log-path-fact/README.rst index d64eec8af..33a36d1da 100644 --- a/roles/set-zuul-log-path-fact/README.rst +++ b/roles/set-zuul-log-path-fact/README.rst @@ -1 +1,12 @@ Sets a fact named ``zuul_log_path`` from zuul variables + +**Role Variables** + +.. zuul:rolevar:: zuul_log_path_shard_build + :type: bool + :default: False + + Flag to specify whether or not paths that include a three character + prefix based on the build uuid should prefix the log path. This is + particularly useful for object storage systems where we want to + spread out the number of files per container. diff --git a/roles/set-zuul-log-path-fact/defaults/main.yaml b/roles/set-zuul-log-path-fact/defaults/main.yaml new file mode 100644 index 000000000..b1abb3a6f --- /dev/null +++ b/roles/set-zuul-log-path-fact/defaults/main.yaml @@ -0,0 +1 @@ +zuul_log_path_shard_build: false diff --git a/roles/set-zuul-log-path-fact/tasks/main.yaml b/roles/set-zuul-log-path-fact/tasks/main.yaml index c49c3a85e..f9dff1b5a 100644 --- a/roles/set-zuul-log-path-fact/tasks/main.yaml +++ b/roles/set-zuul-log-path-fact/tasks/main.yaml @@ -1,14 +1,35 @@ -- name: Set log path for a change - when: zuul.change is defined - set_fact: - zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" +- name: Fileserver friendly log path specifications + when: not zuul_log_path_shard_build + block: + - name: Set log path for a change + when: zuul.change is defined + set_fact: + zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" -- name: Set log path for a ref update - when: zuul.newrev is defined - set_fact: - zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + - name: Set log path for a ref update + when: zuul.newrev is defined + set_fact: + zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" -- name: Set log path for a periodic job - when: zuul.change is not defined and zuul.newrev is not defined - set_fact: - zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + - name: Set log path for a periodic job + when: zuul.change is not defined and zuul.newrev is not defined + set_fact: + zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + +- name: object store friendly log path specifications + when: zuul_log_path_shard_build + block: + - name: Set log path for a change + when: zuul.change is defined + set_fact: + zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + + - name: Set log path for a ref update + when: zuul.newrev is defined + set_fact: + zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + + - name: Set log path for a periodic job + when: zuul.change is not defined and zuul.newrev is not defined + set_fact: + zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}" diff --git a/test-playbooks/base-roles/emit-job-header.yaml b/test-playbooks/base-roles/emit-job-header.yaml index 2dce32de2..51991e1e3 100644 --- a/test-playbooks/base-roles/emit-job-header.yaml +++ b/test-playbooks/base-roles/emit-job-header.yaml @@ -1,4 +1,29 @@ -- name: Test the emit-job-header role +- name: Test the emit-job-header role with swift + hosts: all + roles: + - role: emit-job-header + zuul_log_url: "http://logs.openstack.org" + zuul_log_path_shard_build: true + post_tasks: + # All emit-job-header does is a debug statement so the worst that would + # happen would be that the debug task would fail outright and we'd prevent + # something breaking that debug statement from merging just by running it. + # However, the emit-job-header role includes the set-zuul-log-path-fact + # role. We can only test for zuul_log_path against changes, though. + - name: Assert zuul_log_path by set-zuul-log-path-fact for a change + assert: + that: + - zuul_log_path is defined + - zuul.change in zuul_log_path + - zuul.patchset in zuul_log_path + - zuul.pipeline in zuul_log_path + - zuul.job in zuul_log_path + - zuul.build[:3] == zuul_log_path[:3] + +# Note that the order of these two plays is important. We want the second one +# to run to be the one the creates the correct log path for the currently +# running system. +- name: Test the emit-job-header role without swift hosts: all roles: - role: emit-job-header @@ -17,3 +42,4 @@ - zuul.patchset in zuul_log_path - zuul.pipeline in zuul_log_path - zuul.job in zuul_log_path + - zuul.build[:3] != zuul_log_path[:3]