From 570ca85cd8ea27139226fe279b9c82309cad59ea Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Wed, 17 Feb 2021 15:23:19 +1100
Subject: [PATCH] gerrit: add mariadb_container option

This adds a local mariadb container to the gerrit host to hold the
accountPatchReviewDb database.  This is inspired by a few things

 - since migration to NoteDB, there is only one table left where
   Gerrit records what files have been reviewed for a change.  This
   logically scales with the number of reviews users are doing.
   Pulling the stats on this, we can see since the NoteDB upgrade this
   went from a very busy database (~300 queries/70 commits per second)
   to barely registering one hit per second :
   https://imgur.com/a/QGJV7Fw

   Thus separating the db to an external host for performance reasons
   is not a large concern any more.

 - emperically we've done a bad job in keeping the existing hosted db
   up-to-date; it's still running mysql 5.1 and we have been hit by
   bugs such as the one referenced in-line which silently drops
   backups.

 - The other gerrit option is to use an on-disk H2 database.  This is
   certainly an option, however you need special tools to interact
   with it for migration, etc. and it's not safe to backup from files
   on disk (as opposed to mysqldump).  Upstream advice is unclear, and
   varies between H2 being a performance bottleneck to this being
   ephemeral data that users don't care about.  We know how to admin
   mariadb/mysql and this allows us to migrate and backup data, so
   seems like the best choice.

 - we have a pressing need to update the server to a new operating
   system.  Running the db alongside the gerrit instance minimises
   fiddling we have to do manging connections to and migrating the
   hosted db systems.

 - related to that, we are tending towards more provider independence
   for control-plane servers.  A hosted database product is not always
   provided, so this gives us more flexibility in moving things
   around.

 - the main concern here is memory usage.  "docker stats" reports a
   quiescent container, freshly started on a 8GB host:

    gerrit-compose_mariadb_1  67.32MiB

   After loading a copy of the production table, and then dumping it
   back to a file the same container reports:

    gerrit-compose_mariadb_1  462.6MiB

The existing remote mysql configuration path remains mostly the same.
We move the gerrit startup into a script rather than a CMD so we can
call it after a "wait for db" script in the mariadb_container case
(this is the reccommeded way to enforce ordering [1]).

Backups of the local container need different dump commands; backups
are relocated to a new file and updated.

Testing is converted to use this rather than a local H2 database.

[1] https://docs.docker.com/compose/startup-order/

Change-Id: Iec981ef3c2e38889f91e9759e66295dbfb499c2e
---
 docker/gerrit/base/Dockerfile                 |  12 +-
 .../host_vars/review01.openstack.org.yaml     |   1 +
 .../host_vars/review02.opendev.org.yaml       |   7 +-
 playbooks/roles/gerrit/README.rst             |  31 +++
 playbooks/roles/gerrit/defaults/main.yaml     |   4 +-
 playbooks/roles/gerrit/tasks/backup.yaml      |  43 ++++
 playbooks/roles/gerrit/tasks/main.yaml        |  52 ++---
 .../gerrit/templates/docker-compose.yaml.j2   |  25 +++
 .../root.my.cnf.mariadb_container.j2          |   7 +
 .../{root.my.cnf.j2 => root.my.cnf.mysql.j2}  |   0
 .../roles/gerrit/templates/secure.config.j2   |   6 +-
 playbooks/zuul/gerrit/base.yaml               |  11 +
 playbooks/zuul/gerrit/files/run-gerrit.sh     |   4 +
 playbooks/zuul/gerrit/files/wait-for-it.sh    | 212 ++++++++++++++++++
 .../zuul/templates/group_vars/review.yaml.j2  |   9 +-
 zuul.d/docker-images/gerrit.yaml              |   3 +
 16 files changed, 387 insertions(+), 40 deletions(-)
 create mode 100644 playbooks/roles/gerrit/tasks/backup.yaml
 create mode 100644 playbooks/roles/gerrit/templates/root.my.cnf.mariadb_container.j2
 rename playbooks/roles/gerrit/templates/{root.my.cnf.j2 => root.my.cnf.mysql.j2} (100%)
 create mode 100644 playbooks/zuul/gerrit/base.yaml
 create mode 100755 playbooks/zuul/gerrit/files/run-gerrit.sh
 create mode 100755 playbooks/zuul/gerrit/files/wait-for-it.sh

diff --git a/docker/gerrit/base/Dockerfile b/docker/gerrit/base/Dockerfile
index a2df44ffd5..349da2e814 100644
--- a/docker/gerrit/base/Dockerfile
+++ b/docker/gerrit/base/Dockerfile
@@ -47,6 +47,12 @@ RUN addgroup gerrit --gid 3000 --system \
     --ingroup gerrit \
     gerrit
 
+# Startup scripts
+COPY wait-for-it.sh /wait-for-it.sh
+RUN chmod +x /wait-for-it.sh
+COPY run-gerrit.sh /run-gerrit.sh
+RUN chmod +x /run-gerrit.sh
+
 USER gerrit
 RUN mkdir /var/gerrit/bin \
   && mkdir /var/gerrit/hooks \
@@ -54,7 +60,8 @@ RUN mkdir /var/gerrit/bin \
 
 # Download mysql-connector so that gerrit doens't download it during init.
 RUN mkdir /var/gerrit/lib && \
-  wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.43/mysql-connector-java-5.1.43.jar -O /var/gerrit/lib/mysql-connector-java.jar
+  wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.43/mysql-connector-java-5.1.43.jar -O /var/gerrit/lib/mysql-connector-java.jar && \
+  wget https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.7.2/mariadb-java-client-2.7.2.jar -O /var/gerrit/lib/mariadb-java-client.jar
 
 # Allow incoming traffic
 # OpenDev Gerrit listens on 8081 not default of 8080
@@ -70,5 +77,4 @@ ENV JAVA_OPTIONS ""
 
 # Ulimits should be set on command line or in docker-compose.yaml
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-# The /dev/./urandom is not a typo. https://stackoverflow.com/questions/58991966/what-java-security-egd-option-is-for
-CMD /usr/local/openjdk-11/bin/java -Djava.security.egd=file:/dev/./urandom ${JAVA_OPTIONS} -jar /var/gerrit/bin/gerrit.war daemon -d /var/gerrit
+CMD "/run-gerrit.sh"
diff --git a/inventory/service/host_vars/review01.openstack.org.yaml b/inventory/service/host_vars/review01.openstack.org.yaml
index cec2216945..67bf6be7dd 100644
--- a/inventory/service/host_vars/review01.openstack.org.yaml
+++ b/inventory/service/host_vars/review01.openstack.org.yaml
@@ -60,6 +60,7 @@ gerrit_vhost_name: review.opendev.org
 gerrit_serverid: 4a232e18-c5a9-48ee-94c0-e04e7cca6543
 gerrit_redirect_vhost: review.openstack.org
 gerrit_heap_limit: 48g
+gerrit_reviewdb_database_type: "{{ gerrit_reviewdb_database_override | default('mysql') }}"
 letsencrypt_certs:
   review01-opendev-org-main:
     - review.opendev.org
diff --git a/inventory/service/host_vars/review02.opendev.org.yaml b/inventory/service/host_vars/review02.opendev.org.yaml
index f73c9158d8..8871e2aea6 100644
--- a/inventory/service/host_vars/review02.opendev.org.yaml
+++ b/inventory/service/host_vars/review02.opendev.org.yaml
@@ -4,7 +4,7 @@ gerrit_vhost_name: review.opendev.org
 gerrit_serverid: 4a232e18-c5a9-48ee-94c0-e04e7cca6543
 gerrit_redirect_vhost: review.openstack.org
 gerrit_heap_limit: 48g
-gerrit_database_type: h2
+gerrit_reviewdb_database_type: h2
 letsencrypt_certs:
   review02-opendev-org-main:
     - review.opendev.org
@@ -20,5 +20,6 @@ borg_backup_excludes_extra:
   - /home/gerrit2/review_site/cache/*
   - /home/gerrit2/review_site/tmp/*
   - /home/gerrit2/review_site/index/*
-  # dump directly via stream
-  - /home/gerrit2/mysql_backups/*
+  # live db when used with mariadb_container; dumped by separate job
+  # using mysqldump
+  - /home/gerrit2/reviewdb/*
diff --git a/playbooks/roles/gerrit/README.rst b/playbooks/roles/gerrit/README.rst
index 984e3c5c95..a415076d2d 100644
--- a/playbooks/roles/gerrit/README.rst
+++ b/playbooks/roles/gerrit/README.rst
@@ -1 +1,32 @@
 Run Gerrit.
+
+**Role Variables**
+
+.. zuul:rolevar:: gerrit_reviewdb_database_type
+   :default: h2
+
+   Database to use for the reviewdb
+
+   One of
+
+    * h2 : use local h2 database, not for production
+    * mysql : connect to existing mysql instance
+    * mariadb_container : run a sibling mariadb container
+
+.. zuul:rolevar:: gerrit_reviewdb_mariadb_dbname
+   :default: gerrit
+
+   When ``gerrit_reviewdb_database_type`` is ``mariadb_container``; the default
+   database to make and connect to.
+
+.. zuul:rolevar:: gerrit_reviewdb_mariadb_username
+   :default: gerrit
+
+   When ``gerrit_reviewdb_database_type`` is ``mariadb_container``; the default
+   user to make and connect with.
+
+.. zuul:rolevar:: gerrit_reviewdb_mariadb_password
+   :default: <unset>
+
+   When ``gerrit_reviewdb_database_type`` is ``mariadb_container``; the default
+   password to set for ``gerrit_reviewdb_mariadb_username``
diff --git a/playbooks/roles/gerrit/defaults/main.yaml b/playbooks/roles/gerrit/defaults/main.yaml
index c1cfa1f840..ad8d166380 100644
--- a/playbooks/roles/gerrit/defaults/main.yaml
+++ b/playbooks/roles/gerrit/defaults/main.yaml
@@ -21,7 +21,9 @@ gerrit_container_volumes:
   - /home/gerrit2/review_site/tmp:/var/gerrit/tmp
   - /opt/project-config/gerrit/projects.yaml:/var/gerrit/etc/projects.yaml
   - /opt/project-config/gerrit/projects.ini:/var/gerrit/etc/projects.ini
-gerrit_database_type: MYSQL
 gerrit_project_creator_user: openstack-project-creator
 gerrit_manage_projects_args: "-v"
 gerrit_track_upstream: true
+gerrit_reviewdb_database_type: h2
+gerrit_reviewdb_mariadb_username: gerrit
+gerrit_reviewdb_mariadb_dbname: accountPatchReviewDb
diff --git a/playbooks/roles/gerrit/tasks/backup.yaml b/playbooks/roles/gerrit/tasks/backup.yaml
new file mode 100644
index 0000000000..db74b9441c
--- /dev/null
+++ b/playbooks/roles/gerrit/tasks/backup.yaml
@@ -0,0 +1,43 @@
+- name: Create backup streaming config dir
+  file:
+    path: /etc/borg-streams
+    state: directory
+
+- name: Setup remote mysql backup jobs
+  when: gerrit_reviewdb_database_type == 'mysql'
+  block:
+    # NOTE(ianw) 2021-02-19 We are explicitly backing up just
+    # accountPatchReviewDb because "--all-databases" doesn't work with
+    # our trove instance
+    #  https://bugs.launchpad.net/ubuntu/+source/mysql-5.7/+bug/1914695
+    - name: Create accountPatchReviewDb streaming file
+      copy:
+        content: >-
+            /usr/bin/mysqldump --defaults-file=/root/.gerrit_db.cnf --skip-extended-insert --ignore-table mysql.event --single-transaction --databases accountPatchReviewDb
+        dest: /etc/borg-streams/mysql-accountPatchReviewDb
+
+    - name: Set up cron job to back up gerrit db to disk
+      cron:
+        name: gerrit-backup
+        user: root
+        hour: 0
+        minute: 0
+        job: '/usr/bin/mysqldump --defaults-file=/root/.gerrit_db.cnf --opt --ignore-table mysql.event --single-transaction --databases accountPatchReviewDb | gzip -9 > /home/gerrit2/mysql_backups/gerrit.sql.gz'
+
+# NOTE(ianw) 2021-06-09 : we don't also keep an on-disk backup as we
+# did for the old mysql instance above because it's easy to restore
+# from borg now (in the past with bup it was a real pain to extract
+# data so it was worth managing on-disk copies to possibly avoid that).
+#
+#  # borg-mount <backup-server>
+# the dump file will be under
+#  /opt/backups/review02-mariadb-<date>T<time>/mariadb
+- name: Setup container mariadb backup jobs
+  when: gerrit_reviewdb_database_type == 'mariadb_container'
+  block:
+    - name: Create mariadb streaming file
+      copy:
+        content: >-
+            /usr/local/bin/docker-compose -f /etc/gerrit-compose/docker-compose.yaml exec -T mariadb
+            bash -c '/usr/bin/mysqldump --all-databases --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"'
+        dest: /etc/borg-streams/mariadb
diff --git a/playbooks/roles/gerrit/tasks/main.yaml b/playbooks/roles/gerrit/tasks/main.yaml
index 54fc9c0524..6c9933e098 100644
--- a/playbooks/roles/gerrit/tasks/main.yaml
+++ b/playbooks/roles/gerrit/tasks/main.yaml
@@ -287,12 +287,29 @@
   include_tasks: start.yaml
 
 - name: Set up root mysql conf file
-  when: gerrit_database_type == 'MYSQL'
+  when: gerrit_reviewdb_database_type == 'mysql'
   template:
-    src: root.my.cnf.j2
+    src: root.my.cnf.mysql.j2
     dest: /root/.gerrit_db.cnf
     mode: 0400
 
+- name: Setup mariadb container
+  when: gerrit_reviewdb_database_type == 'mariadb_container'
+  block:
+    - name: Setup reviewdb directory
+      file:
+        state: directory
+        path: /home/gerrit2/reviewdb
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Set up root mariadb conf file
+      template:
+        src: root.my.cnf.mariadb_container.j2
+        dest: /root/.gerrit_db.cnf
+        mode: 0400
+
 - name: Set up cron job to optmize git repos
   cron:
     name: optmize-git-repos
@@ -310,14 +327,6 @@
     minute: 42
     state: "{{ gerrit_track_upstream | bool | ternary('present', 'absent') }}"
 
-- name: Set up cron job to back up gerrit db
-  cron:
-    name: gerrit-backup
-    user: root
-    hour: 0
-    minute: 0
-    job: '/usr/bin/mysqldump --defaults-file=/root/.gerrit_db.cnf --opt --ignore-table mysql.event --single-transaction --databases accountPatchReviewDb | gzip -9 > /home/gerrit2/mysql_backups/gerrit.sql.gz'
-
 # Gerrit rotates their own logs, but doesn't clean them out
 # Delete logs older than a month
 - name: Set up cron job to clean old gerrit logs
@@ -329,24 +338,5 @@
     minute: 1
     hour: 6
 
-- name: Setup db backup streaming job
-  block:
-    - name: Create backup streaming config dir
-      file:
-        path: /etc/borg-streams
-        state: directory
-
-    # NOTE(ianw) 2021-02-09 --all-databases stopped working with
-    # Xenial/our rax version of mysql so we have to backup db's
-    # explicitly.  remove the old call.  This can go after a
-    # deployment run.
-    - name: Remove old stream backup
-      file:
-        state: absent
-        path: /etc/borg-streams/mysql
-
-    - name: Create accountPatchReviewDb streaming file
-      copy:
-        content: >-
-          /usr/bin/mysqldump --defaults-file=/root/.gerrit_db.cnf --skip-extended-insert --ignore-table mysql.event --single-transaction --databases accountPatchReviewDb
-        dest: /etc/borg-streams/mysql-accountPatchReviewDb
+- name: Setup db backups
+  include_tasks: backup.yaml
diff --git a/playbooks/roles/gerrit/templates/docker-compose.yaml.j2 b/playbooks/roles/gerrit/templates/docker-compose.yaml.j2
index aa4553b0ef..7b8569c975 100644
--- a/playbooks/roles/gerrit/templates/docker-compose.yaml.j2
+++ b/playbooks/roles/gerrit/templates/docker-compose.yaml.j2
@@ -1,8 +1,33 @@
 version: '2'
 
+{% if gerrit_reviewdb_database_type == 'mariadb_container' %}
 services:
+  mariadb:
+    image: docker.io/library/mariadb:10.4
+    network_mode: host
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: "{{ gerrit_reviewdb_mariadb_password }}"
+      MYSQL_DATABASE: "{{ gerrit_reviewdb_mariadb_dbname }}"
+      MYSQL_USER: "{{ gerrit_reviewdb_mariadb_username }}"
+      MYSQL_PASSWORD: "{{ gerrit_reviewdb_mariadb_password }}"
+    volumes:
+      # NOTE(ianw) : mounted under /home/gerrit2 (rather than more
+      # usual /var/ in our configs) to keep everything together on the
+      # storage attached at /home/gerrit2 on the server.
+      - /home/gerrit2/reviewdb:/var/lib/mysql
+    logging:
+      driver: syslog
+      options:
+        tag: "docker-mariadb"
+{% endif %}
   gerrit:
     image: {{ gerrit_container_image }}
+{% if gerrit_reviewdb_database_type == 'mariadb_container' %}
+    depends_on:
+      - mariadb
+    command: ["/wait-for-it.sh", "127.0.0.1:3306", "--", "/run-gerrit.sh"]
+{% endif %}
     network_mode: host
     user: gerrit
     stop_signal: SIGHUP
diff --git a/playbooks/roles/gerrit/templates/root.my.cnf.mariadb_container.j2 b/playbooks/roles/gerrit/templates/root.my.cnf.mariadb_container.j2
new file mode 100644
index 0000000000..6128033888
--- /dev/null
+++ b/playbooks/roles/gerrit/templates/root.my.cnf.mariadb_container.j2
@@ -0,0 +1,7 @@
+[client]
+host=127.0.0.1
+port=3306
+user={{ gerrit_reviewdb_mariadb_username }}
+password={{ gerrit_reviewdb_mariadb_password }}
+database={{ gerrit_reviewdb_mariadb_dbname }}
+ssl-mode=disabled
diff --git a/playbooks/roles/gerrit/templates/root.my.cnf.j2 b/playbooks/roles/gerrit/templates/root.my.cnf.mysql.j2
similarity index 100%
rename from playbooks/roles/gerrit/templates/root.my.cnf.j2
rename to playbooks/roles/gerrit/templates/root.my.cnf.mysql.j2
diff --git a/playbooks/roles/gerrit/templates/secure.config.j2 b/playbooks/roles/gerrit/templates/secure.config.j2
index 6050ac46fb..dc0ed39b94 100644
--- a/playbooks/roles/gerrit/templates/secure.config.j2
+++ b/playbooks/roles/gerrit/templates/secure.config.j2
@@ -1,7 +1,11 @@
-{% if gerrit_database_type == 'MYSQL' %}
+{% if gerrit_reviewdb_database_type == 'mysql' %}
 [accountPatchReviewDb]
 	url = jdbc:mysql://{{ gerrit_mysql_host }}:3306/accountPatchReviewDb?characterSetResults=utf8&characterEncoding=utf8&connectionCollation=utf8_bin&useUnicode=yes&user=gerrit2&password={{ gerrit_mysql_password }}
 {% endif %}
+{% if gerrit_reviewdb_database_type == 'mariadb_container' %}
+[accountPatchReviewDb]
+	url = jdbc:mariadb://127.0.0.1:3306/{{ gerrit_reviewdb_mariadb_dbname }}?sessionVariables=character_set_client=utf8,character_set_results=utf8,character_set_connection=utf8,collation_connection=utf8_unicode_ci,collation_database=utf8_unicode_ci,collation_server=utf8_unicode_ci&user={{ gerrit_reviewdb_mariadb_username }}&password={{ gerrit_reviewdb_mariadb_password }}
+{% endif %}
 [auth]
 	registerEmailPrivateKey = {{ gerrit_email_private_key }}
 {% if gerrit_rest_token_private_key is defined %}
diff --git a/playbooks/zuul/gerrit/base.yaml b/playbooks/zuul/gerrit/base.yaml
new file mode 100644
index 0000000000..34ae12583f
--- /dev/null
+++ b/playbooks/zuul/gerrit/base.yaml
@@ -0,0 +1,11 @@
+- hosts: all
+  tasks:
+    # NOTE(ianw) : we have to copy them in here, because the gerrit
+    # image is built in this context.
+    - name: Copy helpers into jeepyb build context
+      copy:
+        src: '{{ item }}'
+        dest: '/home/zuul/src/opendev.org/opendev/jeepyb'
+      loop:
+        - run-gerrit.sh
+        - wait-for-it.sh
diff --git a/playbooks/zuul/gerrit/files/run-gerrit.sh b/playbooks/zuul/gerrit/files/run-gerrit.sh
new file mode 100755
index 0000000000..753acfeb57
--- /dev/null
+++ b/playbooks/zuul/gerrit/files/run-gerrit.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# The /dev/./urandom is not a typo. https://stackoverflow.com/questions/58991966/what-java-security-egd-option-is-for
+/usr/local/openjdk-11/bin/java -Djava.security.egd=file:/dev/./urandom ${JAVA_OPTIONS} -jar /var/gerrit/bin/gerrit.war daemon -d /var/gerrit
diff --git a/playbooks/zuul/gerrit/files/wait-for-it.sh b/playbooks/zuul/gerrit/files/wait-for-it.sh
new file mode 100755
index 0000000000..aad0dc68b6
--- /dev/null
+++ b/playbooks/zuul/gerrit/files/wait-for-it.sh
@@ -0,0 +1,212 @@
+#!/usr/bin/env bash
+
+# See
+#  https://github.com/vishnubob/wait-for-it
+#  https://tracker.debian.org/pkg/wait-for-it
+
+# The MIT License (MIT)
+# Copyright (c) 2016 Giles Hall
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while true; do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then
+            break;
+        fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then
+            break;
+        fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then
+            break;
+        fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi
diff --git a/playbooks/zuul/templates/group_vars/review.yaml.j2 b/playbooks/zuul/templates/group_vars/review.yaml.j2
index e633d4ec15..90f5182e59 100644
--- a/playbooks/zuul/templates/group_vars/review.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/review.yaml.j2
@@ -84,7 +84,14 @@ gerrit_replication_ssh_rsa_key_contents: |
   edHQJDKx5PktPWsAAAAgbW9yZHJlZEBNb250eXMtTWFjQm9vay1BaXIubG9jYWwBAgM=
   -----END OPENSSH PRIVATE KEY-----
 gerrit_replication_ssh_rsa_pubkey_contents: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQhZQ0z+RVPmOzY2f56N9/PrqDeHftvnagPJyOOXnCd/9N0j+stFWNmavvb8y4dRZ+y6lOJpzPYEahwUUXZHAanz5l5as+VihWq7ldcMxSPnmkC9zr65Z8eNDcM2Bzk8gx5e4DE6OgpWkc6ke9MpwI5dmfW7o53gQZkdSc94TuLr+ZCYUKo7fScsVeE+F9dT0PLyW0zU7c23PzYnkKcrB9ihpQfSfbJj9EAtsA3aA8ZdHt78i5r7+0u0JZxaWoKjkCfYqC8ofbTU61YuUO8TTgNgMC6ZzBmTRdRRRKdGun+m1fqtgIqPSi+iZpKnERgg/hPwY+gqcKh+svW6pgCDhJ gerrit-code-review-replication
-gerrit_database_type: h2
+# NOTE(ianw) : review01.openstack.org uses mysql in production but
+# mariadb for testing in the gate.  While the Zuul test still refers
+# to review01.openstack.org we use this to override and test mariadb
+# in the gate.  This override can go away when we retire
+# review01.openstack.org.
+gerrit_reviewdb_database_override: 'mariadb_container'
+gerrit_reviewdb_database_type: mariadb_container
+gerrit_reviewdb_mariadb_password: password
 gerrit_run_compose_up: true
 gerrit_run_init: true
 gerrit_run_init_dev_mode: true
diff --git a/zuul.d/docker-images/gerrit.yaml b/zuul.d/docker-images/gerrit.yaml
index d0080554fd..4282fe2adc 100644
--- a/zuul.d/docker-images/gerrit.yaml
+++ b/zuul.d/docker-images/gerrit.yaml
@@ -8,6 +8,8 @@
       - opendev/system-config
     requires: python-builder-3.7-container-image
     provides: gerrit-base-container-image
+    pre-run: &gerrit-base_prerun
+      - playbooks/zuul/gerrit/base.yaml
     vars: &gerrit-base_vars
       docker_images:
         - context: docker/gerrit/base
@@ -24,6 +26,7 @@
     parent: system-config-upload-image
     requires: python-builder-3.7-container-image
     provides: gerrit-base-container-image
+    pre-run: *gerrit-base_prerun
     required-projects: *gerrit_base_projects
     vars: *gerrit-base_vars
     files: *gerrit-base_files