From b85282c046e93b2bed9b1b02a051e664a09abca3 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Thu, 27 Jun 2019 14:20:02 +1000
Subject: [PATCH] Move rsync mirror updates to new opendev.org mirror-update
host
This move was prompted by wishing to expose the mirror update logs for
the rsync updates so that debugging problems does not require a root
user (note: not actually done in this change; will be a follow-on).
Rather than start hacking at puppet, the rsync mirror scripts make a
nice delination point for starting an Ansible-first/Bionic update.
Most magic is included in the scripts, so there is not much more to do
than copy them. The host uses the existing kerberos and openafs roles
and copies the key material into place (to be added before merge).
Note the scripts are removed from the extant puppet so we don't have
two updates happening simultaneously. This will also require a manual
clean to remove the cron jobs as a once-off when merging.
The other part of mirror-update is the reprepro based scripts for the
various debuntu repositories. They are left as future work for now.
Testing is added to ensure dependencies and scripts are all in place.
Change-Id: I525ac18b55f0e11b0a541b51fa97ee5d6512bf70
---
.zuul.yaml | 23 ++++
inventory/groups.yaml | 3 +
.../manifests/mirror_update.pp | 129 ------------------
playbooks/roles/mirror-update/README.rst | 15 ++
.../roles/mirror-update/files}/cache-stats.sh | 0
.../mirror-update/files/centos-mirror-update | 0
.../mirror-update/files/epel-mirror-update | 0
.../mirror-update/files/fedora-mirror-update | 0
.../files/opensuse-mirror-update | 0
.../files/yum-puppetlabs-mirror-update | 0
playbooks/roles/mirror-update/tasks/main.yaml | 27 ++++
.../roles/mirror-update/tasks/rsync.yaml | 56 ++++++++
playbooks/service-mirror-update.yaml | 11 ++
playbooks/zuul/run-base.yaml | 1 +
.../mirror-update01.opendev.org.yaml.j2 | 12 ++
testinfra/test_mirror-update.py | 50 +++++++
16 files changed, 198 insertions(+), 129 deletions(-)
create mode 100644 playbooks/roles/mirror-update/README.rst
rename {modules/openstack_project/files/mirror => playbooks/roles/mirror-update/files}/cache-stats.sh (100%)
mode change 100644 => 100755
rename modules/openstack_project/files/mirror/centos-mirror-update.sh => playbooks/roles/mirror-update/files/centos-mirror-update (100%)
rename modules/openstack_project/files/mirror/epel-mirror-update.sh => playbooks/roles/mirror-update/files/epel-mirror-update (100%)
rename modules/openstack_project/files/mirror/fedora-mirror-update.sh => playbooks/roles/mirror-update/files/fedora-mirror-update (100%)
mode change 100644 => 100755
rename modules/openstack_project/files/mirror/opensuse-mirror-update.sh => playbooks/roles/mirror-update/files/opensuse-mirror-update (100%)
mode change 100644 => 100755
rename modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh => playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update (100%)
create mode 100644 playbooks/roles/mirror-update/tasks/main.yaml
create mode 100644 playbooks/roles/mirror-update/tasks/rsync.yaml
create mode 100644 playbooks/service-mirror-update.yaml
create mode 100644 playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2
create mode 100644 testinfra/test_mirror-update.py
diff --git a/.zuul.yaml b/.zuul.yaml
index 388d297a15..493c941b04 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -571,6 +571,27 @@
host_copy_output:
'/var/log/apache2/': logs
+- job:
+ name: system-config-run-mirror-update
+ parent: system-config-run
+ description: |
+ Run the playbook for a mirror update node
+ nodeset:
+ nodes:
+ - name: bridge.openstack.org
+ label: ubuntu-bionic
+ - name: mirror-update01.opendev.org
+ label: ubuntu-bionic
+ vars:
+ run_playbooks:
+ - playbooks/service-mirror-update.yaml
+ files:
+ - .zuul.yaml
+ - roles/
+ - playbooks/roles/mirror-update/
+ - playbooks/service-mirror-update.yaml
+ - testinfra/test_mirror-update.py
+
- job:
name: system-config-run-docker-registry
parent: system-config-run
@@ -710,6 +731,7 @@
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-mirror
+ - system-config-run-mirror-update
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
@@ -737,6 +759,7 @@
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-mirror
+ - system-config-run-mirror-update
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
diff --git a/inventory/groups.yaml b/inventory/groups.yaml
index a79c772b8c..62e82cf6c9 100644
--- a/inventory/groups.yaml
+++ b/inventory/groups.yaml
@@ -5,6 +5,7 @@ groups:
afs-client:
- review-dev[0-9]*.open*.org
- mirror[0-9]*.open*.org
+ - mirror-update[0-9]*.opendev.org
- files[0-9]*.open*.org
- ze[0-9]*.open*.org
- afsdb*.open*.org
@@ -67,6 +68,8 @@ groups:
- mirror[0-9]*.openstack.org
mirror_opendev:
- mirror[0-9]*.opendev.org
+ mirror-update:
+ - mirror-update[0-9]*.opendev.org
nodepool:
- nb[0-9]*.open*.org
- nl[0-9]*.open*.org
diff --git a/modules/openstack_project/manifests/mirror_update.pp b/modules/openstack_project/manifests/mirror_update.pp
index 435723b4a9..b1adeaa064 100644
--- a/modules/openstack_project/manifests/mirror_update.pp
+++ b/modules/openstack_project/manifests/mirror_update.pp
@@ -373,135 +373,6 @@ class openstack_project::mirror_update (
]
}
- ### RDO mirror ###
- file { '/etc/rdo.keytab':
- ensure => absent,
- }
-
- file { '/usr/local/bin/rdo-mirror-update':
- ensure => absent,
- }
-
- cron { 'rdo mirror':
- ensure => absent,
- }
-
- ### EPEL mirror ###
- file { '/etc/epel.keytab':
- owner => 'root',
- group => 'root',
- mode => '0400',
- content => $epel_keytab,
- }
-
- file { '/usr/local/bin/epel-mirror-update':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0755',
- source => 'puppet:///modules/openstack_project/mirror/epel-mirror-update.sh',
- }
-
- cron { 'epel mirror':
- user => 'root',
- minute => fqdn_rand(45, 'epel-mirror'),
- hour => '*/2',
- command => 'flock -n /var/run/epel-mirror.lock epel-mirror-update mirror.epel >>/var/log/epel-mirror.log 2>&1',
- environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
- require => [
- File['/usr/local/bin/epel-mirror-update'],
- File['/etc/afsadmin.keytab'],
- File['/etc/epel.keytab'],
- ]
- }
-
- ### Puppetlabs / CentOS mirror ###
- file { '/etc/yum-puppetlabs.keytab':
- owner => 'root',
- group => 'root',
- mode => '0400',
- content => $yum_puppetlabs_keytab,
- }
-
- file { '/usr/local/bin/yum-puppetlabs-mirror-update':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0755',
- source => 'puppet:///modules/openstack_project/mirror/yum-puppetlabs-mirror-update.sh',
- }
-
- cron { 'yum-puppetlabs mirror':
- user => 'root',
- minute => fqdn_rand(45, 'yum-puppetlabs'),
- hour => '*/2',
- command => 'flock -n /var/run/yum-puppetlabs-mirror.lock yum-puppetlabs-mirror-update mirror.yum-puppetlabs >>/var/log/yum-puppetlabs-mirror.log 2>&1',
- environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
- require => [
- File['/usr/local/bin/yum-puppetlabs-mirror-update'],
- File['/etc/afsadmin.keytab'],
- File['/etc/yum-puppetlabs.keytab'],
- ]
- }
-
- ### Fedora mirror ###
- file { '/etc/fedora.keytab':
- owner => 'root',
- group => 'root',
- mode => '0400',
- content => $fedora_keytab,
- }
-
- file { '/usr/local/bin/fedora-mirror-update':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0755',
- source => 'puppet:///modules/openstack_project/mirror/fedora-mirror-update.sh',
- }
-
- cron { 'fedora mirror':
- user => 'root',
- minute => fqdn_rand(45, 'fedora-mirror'),
- hour => '*/2',
- command => 'flock -n /var/run/fedora-mirror.lock fedora-mirror-update mirror.fedora >>/var/log/fedora-mirror.log 2>&1',
- environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
- require => [
- File['/usr/local/bin/fedora-mirror-update'],
- File['/etc/afsadmin.keytab'],
- File['/etc/fedora.keytab'],
- ]
- }
-
- ### openSUSE mirror ###
- file { '/etc/opensuse.keytab':
- owner => 'root',
- group => 'root',
- mode => '0400',
- content => $opensuse_keytab,
- }
-
- file { '/usr/local/bin/opensuse-mirror-update':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0755',
- source => 'puppet:///modules/openstack_project/mirror/opensuse-mirror-update.sh',
- }
-
- cron { 'opensuse mirror':
- user => 'root',
- minute => fqdn_rand(45, 'opensuse-mirror'),
- hour => '*/2',
- command => 'flock -n /var/run/opensuse-mirror.lock opensuse-mirror-update mirror.opensuse >>/var/log/opensuse-mirror.log 2>&1',
- environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
- require => [
- File['/usr/local/bin/opensuse-mirror-update'],
- File['/etc/afsadmin.keytab'],
- File['/etc/opensuse.keytab'],
- ]
- }
-
### Ubuntu Cloud Archive Mirror ###
::openstack_project::reprepro { 'ubuntu-cloud-archive-reprepro-mirror':
confdir => '/etc/reprepro/ubuntu-cloud-archive',
diff --git a/playbooks/roles/mirror-update/README.rst b/playbooks/roles/mirror-update/README.rst
new file mode 100644
index 0000000000..abe7e333e0
--- /dev/null
+++ b/playbooks/roles/mirror-update/README.rst
@@ -0,0 +1,15 @@
+mirror-update
+
+This role sets up the ``mirror-update`` host, which does the periodic
+sync of upstream mirrors to the AFS volumes.
+
+It is not intended to be a particularly generic or flexible role, as
+there is usually only one instance of the mirror-update host (to avoid
+conflicting updates).
+
+At this stage, it handles the mirrors that are updated by ``rsync``
+only. It is expected that it will grow to cover mirroring other
+volumes that are currently done by the legacy ``openstack.org`` host
+and managed by puppet.
+
+**Role Variables**
diff --git a/modules/openstack_project/files/mirror/cache-stats.sh b/playbooks/roles/mirror-update/files/cache-stats.sh
old mode 100644
new mode 100755
similarity index 100%
rename from modules/openstack_project/files/mirror/cache-stats.sh
rename to playbooks/roles/mirror-update/files/cache-stats.sh
diff --git a/modules/openstack_project/files/mirror/centos-mirror-update.sh b/playbooks/roles/mirror-update/files/centos-mirror-update
similarity index 100%
rename from modules/openstack_project/files/mirror/centos-mirror-update.sh
rename to playbooks/roles/mirror-update/files/centos-mirror-update
diff --git a/modules/openstack_project/files/mirror/epel-mirror-update.sh b/playbooks/roles/mirror-update/files/epel-mirror-update
similarity index 100%
rename from modules/openstack_project/files/mirror/epel-mirror-update.sh
rename to playbooks/roles/mirror-update/files/epel-mirror-update
diff --git a/modules/openstack_project/files/mirror/fedora-mirror-update.sh b/playbooks/roles/mirror-update/files/fedora-mirror-update
old mode 100644
new mode 100755
similarity index 100%
rename from modules/openstack_project/files/mirror/fedora-mirror-update.sh
rename to playbooks/roles/mirror-update/files/fedora-mirror-update
diff --git a/modules/openstack_project/files/mirror/opensuse-mirror-update.sh b/playbooks/roles/mirror-update/files/opensuse-mirror-update
old mode 100644
new mode 100755
similarity index 100%
rename from modules/openstack_project/files/mirror/opensuse-mirror-update.sh
rename to playbooks/roles/mirror-update/files/opensuse-mirror-update
diff --git a/modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh b/playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update
similarity index 100%
rename from modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh
rename to playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update
diff --git a/playbooks/roles/mirror-update/tasks/main.yaml b/playbooks/roles/mirror-update/tasks/main.yaml
new file mode 100644
index 0000000000..7e124d453f
--- /dev/null
+++ b/playbooks/roles/mirror-update/tasks/main.yaml
@@ -0,0 +1,27 @@
+# NOTE(ianw) : this does not feel like a a great way to write out
+# binary data. But you can't do what you'd logically think at first
+# with like
+#
+# copy:
+# content: {{ string | b64decode }}
+#
+# because jinja treats the content as utf-8, and ends up mangling
+# "real" binary data like a keytab. See issues like:
+# https://github.com/ansible/ansible/issues/20150
+- name: Install afsadmin keytab
+ shell: 'echo {{ mirror_update_keytab_afsadmin }} | base64 -d > /etc/afsadmin.keytab'
+ args:
+ creates: /etc/afsadmin.keytab
+#no_log: True
+
+- name: Ensure permissions on afsadmin keytab
+ file:
+ path: '/etc/afsadmin.keytab'
+ owner: root
+ group: root
+ mode: '0400'
+
+- name: Setup rsync mirror scripts
+ include_tasks: rsync.yaml
+
+# TODO: reprepro and other mirror components
diff --git a/playbooks/roles/mirror-update/tasks/rsync.yaml b/playbooks/roles/mirror-update/tasks/rsync.yaml
new file mode 100644
index 0000000000..d53ecce229
--- /dev/null
+++ b/playbooks/roles/mirror-update/tasks/rsync.yaml
@@ -0,0 +1,56 @@
+# Mirror scripts that use rsync
+
+- name: Create rsync log output directory
+ file:
+ path: /var/log/rsync-mirrors
+ state: directory
+ owner: root
+ group: root
+ mode: '0755'
+
+- name: Set update script names
+ set_fact:
+ rsync_update_scripts:
+ - centos
+ - epel
+ - fedora
+ - opensuse
+ - yum-puppetlabs
+
+- name: Copy keytab files in place
+ shell: 'echo {{ lookup("vars", "mirror_update_keytab_" + item) }} | base64 -d > /etc/{{ item }}.keytab'
+ args:
+ creates: '/etc/{{ item }}.keytab'
+ loop: '{{ rsync_update_scripts }}'
+# no_log: True
+
+- name: Ensure keytab permissions
+ file:
+ path: '/etc/{{ item }}.keytab'
+ owner: root
+ group: root
+ mode: '0400'
+ loop: '{{ rsync_update_scripts }}'
+
+- name: Copy rsync mirror scripts in place
+ copy:
+ src: '{{ item }}-mirror-update'
+ dest: '/usr/local/bin/{{ item }}-mirror-update'
+ mode: '0755'
+ loop: '{{ rsync_update_scripts }}'
+
+- name: Install update cron jobs
+ cron:
+ name: '{{ item }} mirror sync'
+ state: present
+ job: 'flock -n /var/run/{{ item }}-mirror.lock {{ item }}-mirror-update mirror.{{ item }} >> /var/log/rsync-mirrors/{{ item }}.log 2>&1'
+ hour: '*/2'
+ minute: '{{ 45 | random(seed=inventory_hostname) }}'
+ loop: '{{ rsync_update_scripts }}'
+
+- name: Install logrotate rules
+ include_role:
+ name: logrotate
+ vars:
+ logrotate_file_name: '/var/log/rsync-mirrors/{{ item }}.log'
+ loop: '{{ rsync_update_scripts }}'
\ No newline at end of file
diff --git a/playbooks/service-mirror-update.yaml b/playbooks/service-mirror-update.yaml
new file mode 100644
index 0000000000..24865ca433
--- /dev/null
+++ b/playbooks/service-mirror-update.yaml
@@ -0,0 +1,11 @@
+- hosts: "mirror-update:!disabled"
+ name: "Configure mirror-update"
+ roles:
+ - role: kerberos-client
+ kerberos_realm: 'OPENSTACK.ORG'
+ kerberos_admin_server: 'kdc.openstack.org'
+ kerberos_kdcs:
+ - kdc03.openstack.org
+ - kdc04.openstack.org
+ - role: openafs-client
+ - role: mirror-update
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index 10a918324b..e412df1c1a 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -83,6 +83,7 @@
- host_vars/letsencrypt01.opendev.org.yaml
- host_vars/letsencrypt02.opendev.org.yaml
- host_vars/mirror01.openafs.provider.opendev.org.yaml
+ - host_vars/mirror-update01.opendev.org.yaml
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'
- name: Run base.yaml
diff --git a/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2
new file mode 100644
index 0000000000..8b5bfd472f
--- /dev/null
+++ b/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2
@@ -0,0 +1,12 @@
+mirror_update_keytab_afsadmin: |-
+ AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_centos: |-
+ AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_epel: |-
+ AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_fedora: |-
+ AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_opensuse: |-
+ AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_yum-puppetlabs: |-
+ AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
diff --git a/testinfra/test_mirror-update.py b/testinfra/test_mirror-update.py
new file mode 100644
index 0000000000..9eb762dfa0
--- /dev/null
+++ b/testinfra/test_mirror-update.py
@@ -0,0 +1,50 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+testinfra_hosts = ['mirror-update01.opendev.org']
+
+# Manually calculated from the "secret" value in the test host vars
+KEYTAB_SHA256 = '8f4e9384338ffa41b927ed3c15463512384cb7268693a7c60c1e1254f690b7d0'
+
+def test_tools(host):
+ f = host.file('/usr/bin/k5start')
+ assert f.exists
+ f = host.file('/usr/bin/rsync')
+ assert f.exists
+ f = host.file('/usr/bin/vos')
+ assert f.exists
+
+def test_rsync_scripts(host):
+ for script in ['centos',
+ 'epel',
+ 'fedora',
+ 'opensuse',
+ 'yum-puppetlabs']:
+ f = host.file('/usr/local/bin/%s-mirror-update' % script)
+ assert f.exists
+
+def test_keytabs(host):
+ for keytab in ['/etc/afsadmin.keytab',
+ '/etc/centos.keytab',
+ '/etc/epel.keytab',
+ '/etc/fedora.keytab',
+ '/etc/opensuse.keytab',
+ '/etc/yum-puppetlabs.keytab']:
+
+ f = host.file(keytab)
+ assert f.exists
+ assert f.sha256sum == KEYTAB_SHA256
+ assert f.mode == 0o400
+