diff --git a/playbooks/zuul/run-production-bootstrap-bridge.yaml b/playbooks/zuul/run-production-bootstrap-bridge.yaml
index e833fc4514..52df01e683 100644
--- a/playbooks/zuul/run-production-bootstrap-bridge.yaml
+++ b/playbooks/zuul/run-production-bootstrap-bridge.yaml
@@ -3,3 +3,11 @@
- add-bastion-host
- import_playbook: ../bootstrap-bridge.yaml
+
+- hosts: localhost
+ tasks:
+ - name: Wait for child jobs
+ zuul_return:
+ data:
+ zuul:
+ pause: true
diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml
index 23360b6795..9d1014928a 100644
--- a/zuul.d/infra-prod.yaml
+++ b/zuul.d/infra-prod.yaml
@@ -2,14 +2,59 @@
# in projects.yaml because it's easier to keep an overall view of
# what's happening in there.
-# Make sure only one run of a system-config playbook happens at a time
+# Make sure only one run happens at a time. The deploy pipeline
+# should keep things in order, but this is to stop perodic jobs
+# jumping in.
- semaphore:
- name: infra-prod-playbook
+ name: infra-prod-deployment
+ max: 1
+
+# This semaphore limits the total number of production playbook
+# jobs that can run on bridge at one time. We want things to run in
+# parallel but we have a lot of jobs (particularly in the periodic
+# pipeline) that we don't want to run all at once.
+- semaphore:
+ name: infra-prod-playbook-limit
+ # TODO(clarkb) this semaphore allows us to stage the rollout of
+ # parallel infra-prod job exceution in two steps. First we reorganize
+ # everything but roughly keep the same behaviors as before (max: 1).
+ # When we are happy with that we can bump this to 2 or higher and see
+ # things run in parallel.
max: 1
+- job:
+ name: infra-prod-bootstrap-bridge
+ parent: opendev-infra-prod-setup-src
+ semaphores: infra-prod-deployment
+ description: |
+ Configure the bastion host (bridge)
+
+ This job does minimal configuration on the bastion host
+ (bridge.openstack.org) to allow it to run system-config
+ playbooks against our production hosts. It sets up Ansible
+ and root keys on the host. It also synchronizes the
+ system-config repo from the executor to the bastion.
+
+ Note that this is separate to infra-prod-service-bridge;
+ bridge in it's role as the bastion host actaully runs that
+ against itself; it includes things not strictly needed to make
+ the host able to deploy system-config.
+
+ This job is the parent of all deployment jobs, and will pause
+ until they finish. This prevents conflicts between deployment
+ jobs from changes and periodic runs (which use HEAD of
+ master).
+ run: playbooks/zuul/run-production-bootstrap-bridge.yaml
+ # Do not set file matchers on this job. We must always run this job
+ # before any other infra-prod jobs to ensure system-config is up to
+ # date on bridge before we run our playbooks.
+ nodeset:
+ nodes: []
+
- job:
name: infra-prod-playbook
- parent: opendev-infra-prod-base
+ parent: opendev-infra-prod-setup-keys
+ semaphores: infra-prod-playbook-limit
description: |
Run specified playbook against productions hosts.
@@ -19,7 +64,6 @@
/home/zuul/src/opendev.org/opendev/system-config/playbooks
on the bastion host.
abstract: true
- semaphores: infra-prod-playbook
run: playbooks/zuul/run-production-playbook.yaml
post-run: playbooks/zuul/run-production-playbook-post.yaml
required-projects:
@@ -31,41 +75,6 @@
nodeset:
nodes: []
-- job:
- name: infra-prod-bootstrap-bridge
- parent: opendev-infra-prod-setup-src
- description: |
- Configure the bastion host (bridge)
-
- This job does minimal configuration on the bastion host
- (bridge.openstack.org) to allow it to run system-config
- playbooks against our production hosts. It sets up Ansible
- and root keys on the host. It also synchronizes the system-config
- repo from the executor to the bastion. This is necessary to
- emit an up to date known_hosts file when adding new hosts to
- the inventory.
-
- Note that this is separate to infra-prod-service-bridge;
- bridge in it's role as the bastion host actaully runs that
- against itself; it includes things not strictly needed to make
- the host able to deploy system-config.
- # While we don't run the infra-prod-playbook in this job we do run
- # system-config git repo updates. Until we're ready to stop running
- # system-config updates in every job we use this semaphore to ensure
- # exclusivity.
- semaphores: infra-prod-playbook
- run: playbooks/zuul/run-production-bootstrap-bridge.yaml
- files:
- - playbooks/bootstrap-bridge.yaml
- - playbooks/zuul/run-production-bootstrap-bridge.yaml
- - playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml
- - playbooks/roles/install-ansible/
- - playbooks/roles/root-keys/
- - inventory/base/hosts.yaml
- - inventory/service/group_vars/bastion.yaml
- nodeset:
- nodes: []
-
- job:
name: infra-prod-base
parent: infra-prod-playbook
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 3c386fa484..561005552d 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -348,14 +348,19 @@
- infra-prod-bootstrap-bridge
# From now on, all jobs should depend on base
+ # infra-prod-bootstrap-bridge is a hard dependency of all jobs
+ # because we require the bootstrap job to have run before we
+ # start any playbook jobs, otherwise our buildset would not
+ # hold the bridge semaphore and we may not have the correct
+ # system-config state on bridge.
- infra-prod-base: &infra-prod-base
dependencies:
- name: infra-prod-bootstrap-bridge
- soft: true
# Legacy puppet hosts
- infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
@@ -365,21 +370,25 @@
- infra-prod-service-bridge: &infra-prod-service-bridge
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-run-cloud-launcher: &infra-prod-run-cloud-launcher
dependencies:
# depends on the cloud config written out by
# service-bridge
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-bridge
soft: true
- infra-prod-service-kerberos: &infra-prod-service-kerberos
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-service-afs: &infra-prod-service-afs
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
# NOTE(ianw) in theory we'd want auth changes before
@@ -391,11 +400,13 @@
- infra-prod-service-nameserver: &infra-prod-service-nameserver
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-service-mirror-update: &infra-prod-service-mirror-update
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
@@ -404,6 +415,7 @@
#
- infra-prod-service-gitea-lb: &infra-prod-service-gitea-lb
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: system-config-promote-image-haproxy-statsd
@@ -411,10 +423,12 @@
- infra-prod-service-zuul-db: &infra-prod-service-zuul-db
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: system-config-promote-image-haproxy-statsd
@@ -429,6 +443,7 @@
# role to work.
- infra-prod-service-borg-backup: &infra-prod-service-borg-backup
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
@@ -438,6 +453,7 @@
# this job.
- infra-prod-letsencrypt: &infra-prod-letsencrypt
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: infra-prod-service-nameserver
@@ -446,12 +462,14 @@
# letsencrypt depdencies. keep in alphabetical order
- infra-prod-service-codesearch: &infra-prod-service-codesearch
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-hound
soft: true
- infra-prod-service-eavesdrop: &infra-prod-service-eavesdrop
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@@ -462,6 +480,7 @@
soft: true
- infra-prod-service-etherpad: &infra-prod-service-etherpad
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@@ -470,6 +489,7 @@
soft: true
- infra-prod-service-gitea: &infra-prod-service-gitea
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@@ -478,22 +498,27 @@
soft: true
- infra-prod-service-grafana: &infra-prod-service-grafana
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-graphite: &infra-prod-service-graphite
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-keycloak: &infra-prod-service-keycloak
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-meetpad: &infra-prod-service-meetpad
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-lists3: &infra-prod-service-lists3
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@@ -502,28 +527,34 @@
soft: true
- infra-prod-service-mirror: &infra-prod-service-mirror
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-nodepool: &infra-prod-service-nodepool
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-static: &infra-prod-service-static
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-paste: &infra-prod-service-paste
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-registry: &infra-prod-service-registry
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-refstack: &infra-prod-service-refstack
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@@ -532,6 +563,7 @@
soft: true
- infra-prod-service-review: &infra-prod-service-review
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@@ -540,16 +572,19 @@
soft: true
- infra-prod-service-tracing: &infra-prod-service-tracing
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-zookeeper: &infra-prod-service-zookeeper
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-zookeeper-statsd
soft: true
- infra-prod-service-zuul: &infra-prod-service-zuul
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
@@ -559,6 +594,7 @@
soft: true
- infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-letsencrypt
soft: true
@@ -569,6 +605,7 @@
# accessbot should run on a setup eavesdrop host
- infra-prod-run-accessbot: &infra-prod-run-accessbot
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: infra-prod-service-eavesdrop
@@ -580,6 +617,7 @@
# a setup review host. also sets up gitea
- infra-prod-manage-projects: &infra-prod-manage-projects
dependencies:
+ - name: infra-prod-bootstrap-bridge
- name: infra-prod-base
soft: true
- name: infra-prod-service-review