diff --git a/doc/source/log-roles.rst b/doc/source/log-roles.rst index 1ebcd3e4d..6653b869f 100644 --- a/doc/source/log-roles.rst +++ b/doc/source/log-roles.rst @@ -8,6 +8,7 @@ Log Roles .. zuul:autorole:: fetch-output-openshift .. zuul:autorole:: generate-zuul-manifest .. zuul:autorole:: htmlify-logs +.. zuul:autorole:: local-log-download .. zuul:autorole:: merge-output-to-logs .. zuul:autorole:: publish-artifacts-to-fileserver .. zuul:autorole:: set-zuul-log-path-fact diff --git a/roles/local-log-download/README.rst b/roles/local-log-download/README.rst new file mode 100644 index 000000000..7448fd3d4 --- /dev/null +++ b/roles/local-log-download/README.rst @@ -0,0 +1,11 @@ +Add a script for users to bulk download logs locally + +This adds a script for users to bulk download all logs to their local +system. It queries the Zuul API for the manifest and then copies all +files locally from the log server. + +**Role Variables** + +.. zuul:rolevar:: local_log_download_api + + The Zuul API endpoint to use. Example: ``https://zuul.example.org/api/tenant/{{ zuul.tenant }}`` diff --git a/roles/local-log-download/tasks/main.yaml b/roles/local-log-download/tasks/main.yaml new file mode 100644 index 000000000..0c3cb94b6 --- /dev/null +++ b/roles/local-log-download/tasks/main.yaml @@ -0,0 +1,12 @@ +- name: Check API endpoint is defined + assert: + that: + - local_log_download_api is defined + msg: 'local_log_download_api must be defined' + +- name: Create download script + delegate_to: localhost + template: + dest: '{{ zuul.executor.log_root }}/download-logs.sh' + src: 'download-logs.sh.j2' + mode: 0755 diff --git a/roles/local-log-download/templates/download-logs.sh.j2 b/roles/local-log-download/templates/download-logs.sh.j2 new file mode 100644 index 000000000..7047e1290 --- /dev/null +++ b/roles/local-log-download/templates/download-logs.sh.j2 @@ -0,0 +1,105 @@ +#!/bin/bash + +set -e + +ZUUL_API=${ZUUL_API:-"{{ local_log_download_api }}"} +ZUUL_BUILD_UUID=${ZUUL_BUILD_UUID:-"{{ zuul.build }}"} + +ZUUL_API_URL=${ZUUL_API}/build/${ZUUL_BUILD_UUID} + +(( ${BASH_VERSION%%.*} >= 4 )) || { echo >&2 "bash >=4 required to download."; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo >&2 "Python3 is required to download."; exit 1; } +command -v curl >/dev/null 2>&1 || { echo >&2 "curl is required to download."; exit 1; } + +function log { + echo "$(date -Iseconds) | $@" +} + +{# + # Parse the zuul build results to find the manifest, then parse + # the manifest and print files that shoud be downloaded. The + # first line of output is the base url, then every line after is a + # file to download. +#} +function get_urls { + /usr/bin/env python3 - <<EOF +import gzip +import json +import urllib.request + +base_url = urllib.request.urlopen("${ZUUL_API_URL}").read() +base_json = json.loads(base_url) +manifest_url = [x['url'] for x in base_json['artifacts'] if x.get('metadata', {}).get('type') == 'zuul_manifest'][0] +manifest = urllib.request.urlopen(manifest_url) +if manifest.info().get('Content-Encoding') == 'gzip': + manifest_json = json.loads(gzip.decompress(manifest.read())) +else: + manifest_json = json.loads(manifest.read()) + +def p(node, parent): + if node.get('mimetype') != 'application/directory': + print(parent+node['name']) + if node.get('children'): + for child in node.get('children'): + p(child, parent+node['name']+'/') + +print(base_json['log_url']) +for i in manifest_json['tree']: + p(i, '') + +EOF +} + +function save_file { + local base_url="$1" + local file="$2" + + curl -s --compressed --create-dirs -o "${file}" "${base_url}/${file}" + {# + # Using --compressed we will send an Accept-Encoding: gzip header + # and the data will come to us across the network compressed. + # However, we see weird things like .gz files (as stored on disk) + # actually uncompressed, so we check if this really looks like an + # ASCII file and rename for clarity. + #} + if [[ "${file}" == *.gz ]]; then + local type=$(file "${file}") + if [[ "${type}" =~ "ASCII text" ]] || [[ "${type}" =~ "Unicode text" ]]; then + local new_name=${file%.gz} + log "Renaming to ${new_name}" + mv "${file}" "${new_name}" + fi + fi + +} + +{# + # - read in file _files so we exit if the lookup fails for some reason + # - jinja gets confused with ${#files[@]} as it looks like a comment, + # wrap it in raw +#} +log "Querying ${ZUUL_API_URL} for manifest" +_files="$(get_urls)" +readarray -t files <<< "${_files}" +{% raw %} +len="${#files[@]}" +{% endraw %} +if [[ -z "${DOWNLOAD_DIR}" ]]; then + DOWNLOAD_DIR=$(mktemp -d --tmpdir zuul-logs.XXXXXX) +fi +log "Saving logs to ${DOWNLOAD_DIR}" + +pushd "${DOWNLOAD_DIR}" > /dev/null + +base_url="${files[0]}" +log "Getting logs from ${base_url}" +for (( i=1; i<$len; i++ )); do + file="${files[i]}" + printf -v _out " %-80s [ %04d/%04d ]" "${file}" "${i}" $(( len -1 )) + log "$_out" + save_file $base_url $file +done + +popd >/dev/null + +log "Download complete!" diff --git a/test-playbooks/local-log-download.yaml b/test-playbooks/local-log-download.yaml new file mode 100644 index 000000000..2bf84e7cc --- /dev/null +++ b/test-playbooks/local-log-download.yaml @@ -0,0 +1,21 @@ +- hosts: all + tasks: + + - name: Run local-log-download role + include_role: + name: local-log-download + vars: + local_log_download_api: 'https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}' + + post_tasks: + - name: Check for download script + delegate_to: localhost + file: + path: "{{ zuul.executor.log_root }}/download-logs.sh" + state: file + register: download_script + + - name: Validate download script + assert: + that: + - download_script is not changed diff --git a/zuul-tests.d/logs-jobs.yaml b/zuul-tests.d/logs-jobs.yaml new file mode 100644 index 000000000..0160c8b78 --- /dev/null +++ b/zuul-tests.d/logs-jobs.yaml @@ -0,0 +1,18 @@ +- job: + name: zuul-jobs-test-local-log-download + description: Test the local-log-download role + files: + - roles/local-log-download/.* + run: test-playbooks/local-log-download.yaml + +# -* AUTOGENERATED *- +# The following project section is autogenerated by +# tox -e update-test-platforms +# Please re-run to generate new job lists + +- project: + check: + jobs: &id001 + - zuul-jobs-test-local-log-download + gate: + jobs: *id001