diff --git a/playbooks/service-bridge.yaml b/playbooks/service-bridge.yaml
index 72e0c694b1..164d924773 100644
--- a/playbooks/service-bridge.yaml
+++ b/playbooks/service-bridge.yaml
@@ -32,3 +32,10 @@
- name: Install rackspace DNS backup tool
include_role:
name: rax-dns-backup
+
+ - name: Make ansible log directory
+ file:
+ path: '/var/log/ansible'
+ state: directory
+ owner: root
+ mode: 0755
diff --git a/playbooks/zuul/roles/encrypt-logs/defaults/main.yaml b/playbooks/zuul/roles/encrypt-logs/defaults/main.yaml
new file mode 100644
index 0000000000..445165e3f1
--- /dev/null
+++ b/playbooks/zuul/roles/encrypt-logs/defaults/main.yaml
@@ -0,0 +1,85 @@
+# Anyone who wants to be able to read encrypted published logs should
+# have an entry in this variable in the format
+#
+# - name: <freeform string name>
+# key_id: <key-id of GPG key>
+# gpg_asc: <ASCII-armored PGP public key block>
+#
+encrypt_logs_keys:
+ - name: 'ianw'
+ key_id: '0x9615aec8'
+ gpg_asc: |
+ -----BEGIN PGP PUBLIC KEY BLOCK-----
+ Version: GnuPG v1
+
+ mQINBFQO6lIBEADINS0UyBzR0ljAfYy6h4eh266Hi4R5Q6hw1aZPezz+yrFu+1po
+ XWlHBBGs2BRKakM9vY6JkedX13iYJo0D0TACIQFqICXI91wLLoT4RMwTK5XWhJoq
+ OnMqA578fepgKWivd06lTmd3KwVy6stT/E4i1KWpntHymwiWQ25SOOccGA4cy7qf
+ kGdNDupACXFp5Sa61fBgTLlf9JDR8t4hOqEpb3czDOjSS+LGBdBYzA3DyxBySJk5
+ yhNPug2UNnWv0i+mOuehp1no/VNJzCAvCDjSGLD5fVRBpPCPBGhvr5oJ1VvLU51B
+ 7kuV+pHIurn5dMtocGhgenuwHUouAS+IiYaywlaJ/N9s5GT8jnYGht+DcxyuHFj2
+ ZaTfBwDa3DrSTisKBMwfRMf9zKkwDRERM0C0ReknS1ACwuIlZhDKjlHx1Rv67uuz
+ DIQieX+kThF5YPqFNm52bs4fHfcuKDBU2aCpmmASR8tr15olC8LhW/PlExix8rY7
+ ALUdySejLEdY86xnrqV9g2VSHwdyXDOKMxbwQO5l4IsQgZfdcJJ6KK8sAVvO3Ftc
+ MZzt0oebF3ocO6Azm8de2sx3bVNeod3K1fERG83u7OuAuJmWoHMbnhRqmt/c3Pua
+ ZC4Z+c1mBYe9ZpL/zfUooWoS6H9F9J0tkvoSYXsxSkv3BdGluqiGO0kuXwARAQAB
+ tB1JYW4gV2llbmFuZCA8aWFuQHdpZW5hbmQub3JnPokCOQQTAQgAJAIbAwULCQgH
+ AwUVCgkICwUWAgMBAAIeAQIXgAUCVA7sxQIZAQAKCRC2pvuLlhWuyOQ9D/iYeO7M
+ XojUPYTnAcc7fj2ltXKDhcKmslhB7gqg1y3kLGK8Veys407WAOg124rI3dSViUQ3
+ uEPDBYNc1YSfxIRLPB5tlNYYlAOmuSxyMn7pHBSM/2ZJwAws4H7HrdsDoEJ59/FQ
+ B/T3fhKTAYqYU3OHJNrCTAizLaD4DHQsLB3ZngL2HArRYJS05Kt3f4VnlIPIkCO5
+ lxbLdUvLzgW5fUIRG6zkHJ7Ws9W4b7Lrfmu/HoqNkZVbY6c2uYziiPvi0xX5eVa/
+ EFHerMJVg41Kh3xIAkWmphSMiKHcrAxgJNsiV3LnrbemW4vqsh8HjbdChBkLdd/l
+ jiIsr46oILyry32L5TN1PIVZF5cgsEOmaQXOktT6Ot062j5CJ/uktW2ZThvu75O1
+ q3e6ai5lU9hSLUlyXs0GQUyGz+iW7rQRYkFA0ei0/Kj27h+SJHCvXQlBgDMoYWzz
+ 5P76YN+3GXILA55L56+e7+oowv7OnPJYiNfxsqSmP5DfuI8aZV3e6zowKdAJcQKX
+ modQC0D6HiIAJwfsSbHhCb4S/PmpKxneoKYBophnxTQoUsCgN3ohTj48P8ghHZso
+ GwKxYwj5c8T2MYKtYp8jZFj7e22JrezK6mxNSGRR3kfSxQkSFbMOzhQEc+yBCE5g
+ A1k69gpGnkhNt4LOVyvCw+23SsT1HK04UrWkiEYEEBEIAAYFAlQO7bcACgkQWDlS
+ U/gp6edmbQCdEhGjPxS9ThH956Qu4nXhaEeNBJgAoKbnfznTgEoU0KDJwfjyPeo3
+ p0L5tB1JYW4gV2llbmFuZCA8aWFud0BkZWJpYW4ub3JnPokCNwQTAQgAIQIbAwUL
+ CQgHAwUVCgkICwUWAgMBAAIeAQIXgAUCVA7sxAAKCRC2pvuLlhWuyGUBEACT8QUp
+ OeF7zuZyEb9/g0zBkinv2SXH3DlcGwTnD9XLq6NfmETIM8zAz+WCeFtE2U9cMdP6
+ JAQfhBr49WFYf1fIw6fZvlALSekRLMLlHA6E7P+bERITxahkhsf1cGmNMewgY8SD
+ 7/YDSh4hzaou/2XEPsjOa49kwkylrNmA7+57UbYXINtlc2mlO3vl8ZFaNfRiBnVO
+ fkGGYxTs2h99Apqixw41zx0Jk+ebB+xUdP83aswEwP5CpfADDtJiRORhBUeoqBkv
+ y9UKpwMd6Mtn7+mDDDMI3/NjODsxqxyDDRcHhUhu9mXZJLT1GiaTL2bS5NAn6g07
+ 7n6mvxN58YLizy+TwK1z30Ls1+3WEppjq7RGbXL2fn7qHnddlp0lZOE2P6VcGCJG
+ dfH0tdbaYKWMT7dMBgKRGZIxBw3fToqrTG5fjkmON31A+mloU/JusuYSeOk9H/MZ
+ r8HffeGC/R4ZUpBiNTEvfEkdZGW+WOGFYjjYSBmKw98UpahPW3cmDKaqaDizY6VQ
+ MxcN66eH40arvVID+YQHCkiABgfeHWhP92bIAIIPphmibXg8XMR0gUCTtZjy4usQ
+ 7xqoSvCt/K5Om9OFoSF/CYHB/31r7RzuKQqED+DcIp9QmnLFqeKFcLFEuhHv0llt
+ Gd1m/UQy7NtzfRYZXQmjOpSgrGvwqYJLIe3ProhGBBARCAAGBQJUDu23AAoJEFg5
+ UlP4Kenn748AoLtqkDtDcf9pwupm+PCiDD6xgTb1AKC6zwJFCY+Y5JsBFKp/VUz+
+ 2bA7I7kCDQRUDupSARAAyJbRcgcnwFNvutHiNCnaOgqE6zSV6xTebcPJEqIBlnGP
+ Svk6XLWgbZ6IKbuXIHjn+RggtzDuKnpj+c5QFZpwNuTsJ43t0erIm8FmuvHkUaQl
+ gD/7i4YZulUckN7E41i8BoqD3IcpeEjI/2VrPjhVvzWU+MxRpVrF5VdCWa/tSqFl
+ 8Xr+/nNR7VW0ojr/eplovXaxBXOMf2vraynwny4ftvW7oaJ6KlXuP3BSqpePAd72
+ kz33s9k3X/TeUruTtIn/dWXyT36b67IaZRdzxBHguYcKCy9+r+f1IIssq6kjgZi0
+ o2VijZ2DZvR0YB0cSe1FHKCmaqxx9I+wajBIw/bgFf3gDmABPpTK0Z4T3oIrUGoF
+ q7JXcqT9X+cZidimZSdE6xgiMzS2xSNXf1bmK/6BslU3/PiwMy7MDUM02z1pGKVK
+ xV+5Al7Tf/gFYMrpQ6r6rDAVj9Ube+ORTIfJcWKbBJ1RLIo8NA6/K+QqezMsyX9n
+ IZXM1fdQVm9OsGgzdT8h2kMLzm1JAj7yahAnig33MKp/IUfcsjLsR8GnkMTxv8Tq
+ O8X5AHyxk+sbzSLmFpT6nGGsGL4o+3rfPyQmhWkkuxjBRQeUQABCLgF01tfzlXch
+ JwLygD/ENAeYMDrNaTp9sgWsX7QZvHE/cLLk8bKqtziGRBtsTgBAjIHnKFSJAMEA
+ EQEAAYkCHwQYAQgACQUCVA7qUgIbDAAKCRC2pvuLlhWuyKqRD/4zm/PsFH3njOsJ
+ MrOwidxvlKNG+x6GZc6W6yvO1OyVZVtbsSwVAJfFQZejmj0NSL1sl/isQlQxBjXZ
+ TFjv3sQhUt+J6tOI7SzneP0iFXvhyuz3ffe0myG1pBKCgISmUDPEADYd9D7puyv6
+ sZe56wiMXxHOiTrlHLHGFpmezfgybX2f/sQpZcUP4titLhVCt6A2cliHQcCoQjOI
+ FibNfks3jgql6YtF/JPkThxOcuT5BF59hVoSh41iLlO5feMJYIa8IQk3o7dzSVCX
+ EKotxotuYQ/bvmwZDT50TIJY3GzrpQsLxcM5+jaRB+S4rF91tAQ6msvdaSbhoARH
+ VEBMZAFM4257XMZXghgqdbHgtU8IEpL7rGwuIyoC/pEsLkbnzKuwthWag1U6KmoO
+ wevlH3u9Lj57zJjsn2RTrHHzVNIlIe8tJuInnacAoYq1lsmj5NCrZ8wpTSnv7C+o
+ rPQDmnnQz9MoUruyutd51GCMUTv4KthSYElgS9y18BPT0F5cHWkNBBT+W3NEPjgQ
+ 1SAxJ3dpqZgDlmWBO9XJ5rhb5rUiUDc03Dmnq7qLZtHQEXTipiUkuyoF24hisJ7+
+ XgLPdWwyuOjfSc6foik6xYSuR1duxypmb9BidTmVPtQtG9uRvpc6vc7nRoUJKN8U
+ YyJrcbCb0lGKJnpdWIXnDldS91E0Lw==
+ =jbiD
+ -----END PGP PUBLIC KEY BLOCK-----
+
+# This is the default list of keys from ``encrypt_log_keys`` that wish
+# to always be a recipient of encrypted logs. Others can add
+# themselves to particular prod jobs of interest individually.
+encrypt_logs_recipients:
+ - ianw
+
diff --git a/playbooks/zuul/roles/encrypt-logs/tasks/main.yaml b/playbooks/zuul/roles/encrypt-logs/tasks/main.yaml
new file mode 100644
index 0000000000..ba49cf836b
--- /dev/null
+++ b/playbooks/zuul/roles/encrypt-logs/tasks/main.yaml
@@ -0,0 +1,27 @@
+- name: Encrypt file
+ include_role:
+ name: encrypt-file
+ vars:
+ encrypt_file: '{{ encrypt_logs_files }}'
+ encrypt_file_keys: '{{ encrypt_logs_keys }}'
+ encrypt_file_recipients: '{{ encrypt_logs_recipients + encrypt_logs_job_recipients|default([]) }}'
+
+- name: Write download script
+ template:
+ src: download-logs.sh.j2
+ dest: '{{ encrypt_logs_download_script_path }}/download-logs.sh'
+ mode: 0755
+ vars:
+ encrypt_logs_download_api: 'https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}'
+
+- name: Return artifact
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ # This is parsed by the log download script above, so any
+ # changes to format must be accounted for there too.
+ - name: Encrypted logs
+ url: '{{ encrypt_logs_artifact_path }}'
+ metadata:
+ logfiles: "{{ encrypt_logs_files | map('basename') | map('regex_replace', '^(.*)$', '\\1.gpg') | list }}"
diff --git a/playbooks/zuul/roles/encrypt-logs/templates/download-logs.sh.j2 b/playbooks/zuul/roles/encrypt-logs/templates/download-logs.sh.j2
new file mode 100644
index 0000000000..69816933e0
--- /dev/null
+++ b/playbooks/zuul/roles/encrypt-logs/templates/download-logs.sh.j2
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+set -e
+
+ZUUL_API=${ZUUL_API:-"{{ encrypt_logs_download_api }}"}
+ZUUL_BUILD_UUID=${ZUUL_BUILD_UUID:-"{{ zuul.build }}"}
+{% raw %}
+ZUUL_API_URL=${ZUUL_API}/build/${ZUUL_BUILD_UUID}
+
+(( ${BASH_VERSION%%.*} >= 4 )) || { echo >&2 "bash >=4 required to download."; exit 1; }
+command -v python3 >/dev/null 2>&1 || { echo >&2 "Python3 is required to download."; exit 1; }
+command -v curl >/dev/null 2>&1 || { echo >&2 "curl is required to download."; exit 1; }
+
+function log {
+ echo "$(date -Iseconds) | $@"
+}
+
+function get_urls {
+ /usr/bin/env python3 - <<EOF
+import gzip
+import json
+import urllib.request
+from urllib.error import HTTPError
+import sys
+
+try:
+ base_url = urllib.request.urlopen("${ZUUL_API_URL}").read()
+ base_json = json.loads(base_url)
+ for a in base_json['artifacts']:
+ if a['name'] == 'Encrypted logs':
+ url = a['url']
+ logfiles = (url + '/' + f for f in a['metadata']['logfiles'])
+ for l in logfiles:
+ print(l)
+
+except HTTPError as e:
+ if e.code == 404:
+ print(
+ "Could not find build UUID in Zuul API. This can happen with "
+ "buildsets still running, or aborted ones. Try again after the "
+ "buildset is reported back to Zuul.", file=sys.stderr)
+ else:
+ print(e, file=sys.stderr)
+ sys.exit(2)
+
+EOF
+}
+
+function save_file {
+ local exit_code=0
+ local xtra_args="--compressed"
+
+ curl -s ${xtra_args} -o $(basename "${file}") "${file}" || exit_code=$?
+ if [[ $exit_code -ne 0 ]]; then
+ log "Failed to download ${base_url}${file}"
+ exit 1
+ fi
+}
+
+log "Querying ${ZUUL_API_URL} for manifest"
+_files="$(get_urls)"
+readarray -t files <<< "${_files}"
+
+len="${#files[@]}"
+if [[ -z "${DOWNLOAD_DIR}" ]]; then
+ DOWNLOAD_DIR=$(mktemp -d --tmpdir zuul-logs.XXXXXX)
+fi
+log "Saving logs to ${DOWNLOAD_DIR}"
+
+pushd "${DOWNLOAD_DIR}" > /dev/null
+
+log "Getting logs from ${ZUUL_BUILD_ID}"
+for (( i=1; i<$len; i++ )); do
+ file="${files[i]}"
+ printf -v _out " %-80s [ %04d/%04d ]" "${file}" "${i}" $(( len -1 ))
+ log "$_out"
+ save_file $file
+done
+
+for f in ${DOWNLOAD_DIR}/*.gpg; do
+ log "Decrypting $(basename $f)"
+ gpg --output ${f/.gpg/} --decrypt ${f}
+ rm ${f}
+done
+
+popd >/dev/null
+
+log "Download to ${DOWNLOAD_DIR} complete!"
+{% endraw %}
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index 66a1495baa..d6d7c61a46 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -92,18 +92,35 @@
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'
- name: Run base.yaml
- command: ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/base.yaml
+ shell: 'ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/base.yaml 2>&1 | tee /var/log/ansible/base.yaml.log'
- name: Run bridge service playbook
- command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-bridge.yaml
+ shell: 'ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-bridge.yaml 2>&1 | tee /var/log/ansible/service-bridge.yaml.log'
- name: Run dstat logger playbook
- command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-dstatlogger.yaml
+ shell: 'ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-dstatlogger.yaml 2>&1 | tee /var/log/ansible/service-dstatlogger.yaml.log'
+
- name: Run playbook
when: run_playbooks is defined
loop: "{{ run_playbooks }}"
- command: "ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/{{ item }}"
+ shell: "ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/{{ item }} 2>&1 | tee /var/log/ansible/{{ item | basename }}.log"
+
+ - name: Build list of playbook logs
+ find:
+ paths: '/var/log/ansible'
+ patterns: '*.yaml.log'
+ register: _run_playbooks_logs
+
+ - name: Encrypt playbook logs
+ when: run_playbooks is defined
+ include_role:
+ name: encrypt-logs
+ vars:
+ encrypt_logs_files: '{{ _run_playbooks_logs.files | map(attribute="path") | list }}'
+ encrypt_logs_artifact_path: 'bridge.openstack.org/ansible'
+ encrypt_logs_download_script_path: '/var/log/ansible'
+
- name: Run test playbook
when: run_test_playbook is defined
- shell: "ANSIBLE_ROLES_PATH=/home/zuul/src/opendev.org/opendev/system-config/playbooks/roles ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/{{ run_test_playbook }}"
+ shell: "ANSIBLE_ROLES_PATH=/home/zuul/src/opendev.org/opendev/system-config/playbooks/roles ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/{{ run_test_playbook }} 2>&1 | tee /var/log/ansible/{{ run_test_playbook | basename }}.log"
- name: Generate testinfra extra data fixture
set_fact:
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index 219996f1fc..0cde0543b5 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -28,6 +28,7 @@
'{{ zuul.project.src_dir }}/test-results.html': logs
'{{ zuul.project.src_dir }}/inventory/base/gate-hosts.yaml': logs
'/var/log/screenshots': logs
+ '/var/log/ansible': logs
# Note: the following two jobs implement the variant-based multiple
# inheritance trick. Both of these variants will always apply,