opendev/system-config
Code Issues Proposed changes

Reapply "Bootstrap-bridge as top-level job"

Browse Source 
This reverts commit 03816fa43363d9162749bf3cf418f788acfee7cc.

This is a partial reapplication of the previously broken change. We make
a small edit to the ansible playbook to run zuul_return in a valid
context. Specifically as a task against localhost.

We also move the infra-prod-bootstrap-bridge dependency into the PPC
because the PPC dependencies override job dpendencies.

Change-Id: Icc2e0871abfed28937eb96bc14bb2be6b0d882d8
This commit is contained in:
Clark Boylan 2025-03-05 09:09:31 -08:00 committed by James E. Blair
parent 03816fa433
commit 990e6ce8a3
3 changed files with 95 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
playbooks/zuul/run-production-bootstrap-bridge.yaml
View File

@ -3,3 +3,11 @@
- add-bastion-host
- import_playbook: ../bootstrap-bridge.yaml
- hosts: localhost
tasks:
- name: Wait for child jobs
zuul_return:
data:
zuul:
pause: true

87
zuul.d/infra-prod.yaml
View File

@ -2,14 +2,59 @@
# in projects.yaml because it's easier to keep an overall view of
# what's happening in there.
# Make sure only one run of a system-config playbook happens at a time
# Make sure only one run happens at a time. The deploy pipeline
# should keep things in order, but this is to stop perodic jobs
# jumping in.
- semaphore:
name: infra-prod-playbook
name: infra-prod-deployment
max: 1
# This semaphore limits the total number of production playbook
# jobs that can run on bridge at one time. We want things to run in
# parallel but we have a lot of jobs (particularly in the periodic
# pipeline) that we don't want to run all at once.
- semaphore:
name: infra-prod-playbook-limit
# TODO(clarkb) this semaphore allows us to stage the rollout of
# parallel infra-prod job exceution in two steps. First we reorganize
# everything but roughly keep the same behaviors as before (max: 1).
# When we are happy with that we can bump this to 2 or higher and see
# things run in parallel.
max: 1
- job:
name: infra-prod-bootstrap-bridge
parent: opendev-infra-prod-setup-src
semaphores: infra-prod-deployment
description: |
Configure the bastion host (bridge)
This job does minimal configuration on the bastion host
(bridge.openstack.org) to allow it to run system-config
playbooks against our production hosts. It sets up Ansible
and root keys on the host. It also synchronizes the
system-config repo from the executor to the bastion.
Note that this is separate to infra-prod-service-bridge;
bridge in it's role as the bastion host actaully runs that
against itself; it includes things not strictly needed to make
the host able to deploy system-config.
This job is the parent of all deployment jobs, and will pause
until they finish. This prevents conflicts between deployment
jobs from changes and periodic runs (which use HEAD of
master).
run: playbooks/zuul/run-production-bootstrap-bridge.yaml
# Do not set file matchers on this job. We must always run this job
# before any other infra-prod jobs to ensure system-config is up to
# date on bridge before we run our playbooks.
nodeset:
nodes: []
- job:
name: infra-prod-playbook
parent: opendev-infra-prod-base
parent: opendev-infra-prod-setup-keys
semaphores: infra-prod-playbook-limit
description: |
Run specified playbook against productions hosts.
@ -19,7 +64,6 @@
/home/zuul/src/opendev.org/opendev/system-config/playbooks
on the bastion host.
abstract: true
semaphores: infra-prod-playbook
run: playbooks/zuul/run-production-playbook.yaml
post-run: playbooks/zuul/run-production-playbook-post.yaml
required-projects:
@ -31,41 +75,6 @@
nodeset:
nodes: []
- job:
name: infra-prod-bootstrap-bridge
parent: opendev-infra-prod-setup-src
description: |
Configure the bastion host (bridge)
This job does minimal configuration on the bastion host
(bridge.openstack.org) to allow it to run system-config
playbooks against our production hosts. It sets up Ansible
and root keys on the host. It also synchronizes the system-config
repo from the executor to the bastion. This is necessary to
emit an up to date known_hosts file when adding new hosts to
the inventory.
Note that this is separate to infra-prod-service-bridge;
bridge in it's role as the bastion host actaully runs that
against itself; it includes things not strictly needed to make
the host able to deploy system-config.
# While we don't run the infra-prod-playbook in this job we do run
# system-config git repo updates. Until we're ready to stop running
# system-config updates in every job we use this semaphore to ensure
# exclusivity.
semaphores: infra-prod-playbook
run: playbooks/zuul/run-production-bootstrap-bridge.yaml
files:
- playbooks/bootstrap-bridge.yaml
- playbooks/zuul/run-production-bootstrap-bridge.yaml
- playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml
- playbooks/roles/install-ansible/
- playbooks/roles/root-keys/
- inventory/base/hosts.yaml
- inventory/service/group_vars/bastion.yaml
nodeset:
nodes: []
- job:
name: infra-prod-base
parent: infra-prod-playbook

40
zuul.d/project.yaml
View File

@ -348,14 +348,19 @@
- infra-prod-bootstrap-bridge
# From now on, all jobs should depend on base
# infra-prod-bootstrap-bridge is a hard dependency of all jobs
# because we require the bootstrap job to have run before we
# start any playbook jobs, otherwise our buildset would not
# hold the bridge semaphore and we may not have the correct
# system-config state on bridge.
- infra-prod-base: &infra-prod-base
dependencies:
- name: infra-prod-bootstrap-bridge
soft: true
# Legacy puppet hosts
- infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
@ -365,21 +370,25 @@
- infra-prod-service-bridge: &infra-prod-service-bridge
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-run-cloud-launcher: &infra-prod-run-cloud-launcher
dependencies:
# depends on the cloud config written out by
# service-bridge
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-bridge
soft: true
- infra-prod-service-kerberos: &infra-prod-service-kerberos
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-service-afs: &infra-prod-service-afs
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
# NOTE(ianw) in theory we'd want auth changes before
@ -391,11 +400,13 @@
- infra-prod-service-nameserver: &infra-prod-service-nameserver
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-service-mirror-update: &infra-prod-service-mirror-update
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
@ -404,6 +415,7 @@
#
- infra-prod-service-gitea-lb: &infra-prod-service-gitea-lb
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: system-config-promote-image-haproxy-statsd
@ -411,10 +423,12 @@
- infra-prod-service-zuul-db: &infra-prod-service-zuul-db
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: system-config-promote-image-haproxy-statsd
@ -429,6 +443,7 @@
# role to work.
- infra-prod-service-borg-backup: &infra-prod-service-borg-backup
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
@ -438,6 +453,7 @@
# this job.
- infra-prod-letsencrypt: &infra-prod-letsencrypt
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: infra-prod-service-nameserver
@ -446,12 +462,14 @@
# letsencrypt depdencies. keep in alphabetical order
- infra-prod-service-codesearch: &infra-prod-service-codesearch
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-hound
soft: true
- infra-prod-service-eavesdrop: &infra-prod-service-eavesdrop
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@ -462,6 +480,7 @@
soft: true
- infra-prod-service-etherpad: &infra-prod-service-etherpad
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@ -470,6 +489,7 @@
soft: true
- infra-prod-service-gitea: &infra-prod-service-gitea
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@ -478,22 +498,27 @@
soft: true
- infra-prod-service-grafana: &infra-prod-service-grafana
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-graphite: &infra-prod-service-graphite
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-keycloak: &infra-prod-service-keycloak
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-meetpad: &infra-prod-service-meetpad
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-lists3: &infra-prod-service-lists3
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@ -502,28 +527,34 @@
soft: true
- infra-prod-service-mirror: &infra-prod-service-mirror
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-nodepool: &infra-prod-service-nodepool
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-static: &infra-prod-service-static
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-paste: &infra-prod-service-paste
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-registry: &infra-prod-service-registry
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-refstack: &infra-prod-service-refstack
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@ -532,6 +563,7 @@
soft: true
- infra-prod-service-review: &infra-prod-service-review
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@ -540,16 +572,19 @@
soft: true
- infra-prod-service-tracing: &infra-prod-service-tracing
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-zookeeper: &infra-prod-service-zookeeper
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-zookeeper-statsd
soft: true
- infra-prod-service-zuul: &infra-prod-service-zuul
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@ -559,6 +594,7 @@
soft: true
- infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
@ -569,6 +605,7 @@
# accessbot should run on a setup eavesdrop host
- infra-prod-run-accessbot: &infra-prod-run-accessbot
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: infra-prod-service-eavesdrop
@ -580,6 +617,7 @@
# a setup review host. also sets up gitea
- infra-prod-manage-projects: &infra-prod-manage-projects
dependencies:
- name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: infra-prod-service-review