borg-backup-server: run a weekly backup verification

This checks the backup archives and alerts us if anything seems wrong.
This will take a few hours, so we run once a week.

Change-Id: I832c0d29a37df94d4bf2704c59bb3f8d855c3cc8

playbooks/roles/borg-backup-server/files/verify-borg-backups.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+pushd /opt/backups
+
+for u in borg-*; do
+    BORG_REPO=/opt/backups/$u/backup
+
+    sudo BORG_RELOCATED_REPO_ACCESS_IS_OK=y BORG_REPO=${BORG_REPO} -u ${u} -s  <<'EOF'
+
+    echo "$(date) Verifying ${BORG_REPO} ..."
+    /opt/borg/bin/borg check --verify-data
+    if [[ $? -ne 0 ]]; then
+        echo "$(date) *** Verification failed"
+        echo "Inconsistency found in backup ${BORG_REPO} on $(hostname) at $(date)" |
+            mail -s "ACTION REQUIRED: Backup inconsistency: ${BORG_REPO}" infra-root@openstack.org
+    else
+        echo "$(date) ... done"
+        echo
+    fi
+
+EOF
+done

playbooks/roles/borg-backup-server/tasks/main.yaml

@@ -31,6 +31,29 @@
     minute: '0'
     hour: '0'
 
+- name: Install backup verification
+  copy:
+    src: 'verify-borg-backups.sh'
+    dest: '/usr/local/bin/verify-borg-backups'
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Run backup verification
+  cron:
+    name: verify-borg-backups
+    state: present
+    job: '/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log'
+    minute: '0'
+    hour: '0'
+    weekday: '0'
+
+- name: Rotate verification logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: '/var/log/verify-borg-backups.log'
+
 - name: Build all borg users from backup hosts
   set_fact:
     borg_users: '{{ borg_users }} + [ {{ hostvars[item]["borg_user"] }} ]'

testinfra/test_borg_backups.py

@@ -96,3 +96,11 @@ def test_borg_server_prune(host):
 
     cmd = host.run('echo "prune" | /usr/local/bin/prune-borg-backups &> /var/log/prune-borg-backups.log')
     assert cmd.succeeded
+
+def test_borg_server_verify(host):
+    hostname = host.backend.get_hostname()
+    if hostname.startswith('borg-backup-test'):
+        pytest.skip()
+
+    cmd = host.run('/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log')
+    assert cmd.succeeded

zuul.d/system-config-run.yaml

@@ -360,6 +360,7 @@
       borg-backup01.region.provider.opendev.org:
         host_copy_output:
           '/var/log/prune-borg-backups.log': logs
+          '/var/log/verify-borg-backups.log': logs
       borg-backup-test01.opendev.org:
         host_copy_output:
           '/var/log/borg-backup-borg-backup01.region.provider.opendev.org.log': logs