[CPID-354] Improve MariaDB Backup/Restore validation process

HTK has been updated to 0.2.48 version that includes local backup verification step before uploading to remote storage.

In mysqlclient-utility chart HTK is used to perform test backup/restore processes. This chart has been updated to match HTK changes.
The main difference that a mariadb verification server has to be running as a side container.

In etcdctl-utility chart .Values.manifests.secret_certificates value was set to False and a condition has been added to deployment-etcdctl-utility.yaml to avoid looking for etcdctl-etcd-secrets secret to mount.

Also in mysqlclient-utility in ensure_ondemand_pod_exists() function of dbutils.sh made the folloging adjustment: number of containers in *-ondemand pod in pod existence verification condition set to 2/2. This is because we added the second container to this pod with verification server.

Additionally, bumping up HTK version to 0.2.48 from a commit id obtained from merge of https://review.opendev.org/c/openstack/openstack-helm-infra/+/853027 and set proper commit id in this file: tools/helm_tk.sh

Change-Id: I9d3796500574da3582cf67d8aeeec8c796dd88dd

.gitignore

@@ -75,3 +75,6 @@ tmp/
 
 # Developer symlinks
 helm-toolkit
+
+# VScode IDE
+.vscode

charts/etcdctl-utility/Chart.yaml

@@ -13,4 +13,4 @@
 apiVersion: v1
 description: etcdctl Client
 name: etcdctl-utility
-version: 0.1.0
+version: 0.1.1

charts/etcdctl-utility/templates/deployment-etcdctl-utility.yaml

@@ -131,6 +131,7 @@ spec:
           command:
             - /tmp/bootstrap.sh
           volumeMounts:
+{{- if .Values.manifests.secret_certificates }}
             - name: etcdctl-etcd-secrets
               mountPath: {{ .Values.endpoints.etcd.auth.client.path.ca }}
               subPath: tls.ca
@@ -143,6 +144,7 @@ spec:
               mountPath: {{ .Values.endpoints.etcd.auth.client.path.key }}
               subPath: tls.key
               readOnly: true
+{{- end }}
             - name: etcdctl-utility-bin-utilscli
               mountPath: /tmp/bootstrap.sh
               subPath: bootstrap.sh
@@ -203,10 +205,12 @@ spec:
           configMap:
             name: {{ printf "%s-%s" $envAll.Release.Name "bin-utilscli" }}
             defaultMode: 0755
+{{- if .Values.manifests.secret_certificates }}
         - name: etcdctl-etcd-secrets
           secret:
             secretName: etcdctl-etcd-secrets
             defaultMode: 0444
+{{- end }}
 {{- if .Values.pod.mounts.etcd.container.etcdctl_utility.volumes }}
 {{ .Values.pod.mounts.etcd.container.etcdctl_utility.volumes | toYaml | indent 8 }}
 {{- end }}

charts/etcdctl-utility/values.yaml

@@ -246,4 +246,4 @@ manifests:
   configmap_etc_sudoers: true
   deployment_etcdctl_utility: true
   job_image_repo_sync: false
-  secret_certificates: true
+  secret_certificates: false

charts/mysqlclient-utility/Chart.yaml

@@ -13,4 +13,4 @@
 apiVersion: v1
 description: Porthole MySql Client
 name: mysqlclient-utility
-version: 0.1.0
+version: 0.1.1

charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl

@@ -110,7 +110,7 @@ function ensure_ondemand_pod_exists() {
     CONTAINERS=$(echo "$POD_LISTING" | awk '{print $2}')
     # There should only ever be one ondemand pod existing at any time, so if
     #    we find any which are not ready remove them, even if completed.
-    if [[ $STATUS != "Running" || $CONTAINERS != "1/1" ]]; then
+    if [[ $STATUS != "Running" || $CONTAINERS != "2/2" ]]; then
       echo "Found an old on-demand pod; removing it."
       remove_job "$NAMESPACE" "$ONDEMAND_JOB"
       if [[ $? -ne 0 ]]; then

charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl

@@ -9,7 +9,8 @@ if [[ $MARIADB_POD_NAMESPACE == "" ]]; then
 fi
 
 export MARIADB_CONF_SECRET={{ $envAll.Values.conf.mariadb_backup_restore.secrets.conf_secret }}
-export MARIADB_IMAGE_NAME=$(kubectl get cronjob -n ${MARIADB_POD_NAMESPACE} mariadb-backup -o yaml -o jsonpath="{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{'\n'}{end}" | grep mysqlclient-utility)
+export MARIADB_IMAGE_NAME=$(kubectl get cronjob -n ${MARIADB_POD_NAMESPACE} mariadb-backup -o yaml -o jsonpath="{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{'\n'}{end}" | grep mariadb)
+export MYSQLCLIENT_UTILTIY_IMAGE_NAME=$(kubectl get cronjob -n ${MARIADB_POD_NAMESPACE} mariadb-backup -o yaml -o jsonpath="{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{'\n'}{end}" | grep mysqlclient-utility)
 export MARIADB_BACKUP_BASE_PATH=$(kubectl get secret -n ${MARIADB_POD_NAMESPACE} ${MARIADB_CONF_SECRET} -o json | jq -r .data.BACKUP_BASE_PATH | base64 -d)
 MARIADB_REMOTE_BACKUP_ENABLED=$(kubectl get secret -n ${MARIADB_POD_NAMESPACE} ${MARIADB_CONF_SECRET} -o json | jq -r .data.REMOTE_BACKUP_ENABLED | base64 -d)
 export MARIADB_REMOTE_BACKUP_ENABLED=$(echo $MARIADB_REMOTE_BACKUP_ENABLED | sed 's/"//g')
@@ -19,6 +20,11 @@ if [[ $MARIADB_IMAGE_NAME == "" ]]; then
   exit 1
 fi
 
+if [[ $MYSQLCLIENT_UTILTIY_IMAGE_NAME == "" ]]; then
+  echo "Cannot find the utility image for populating MYSQLCLIENT_UTILTIY_IMAGE_NAME variable."
+  exit 1
+fi
+
 export TMP_FILE=$(mktemp -p /tmp)
 
 if ! kubectl -n ${MARIADB_POD_NAMESPACE} --no-headers=true get secret {{ $envAll.Values.conf.mariadb_backup_restore.secrets.tls_secret }} > /dev/null 2>&1 ; then
@@ -44,7 +50,7 @@ spec:
     metadata:
       annotations:
         {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
-{{ dict "envAll" $envAll "podName" "mariadb-ondemand" "containerNames" (list "ondemand-perms" "mariadb-ondemand" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+{{ dict "envAll" $envAll "podName" "mariadb-ondemand" "containerNames" (list "ondemand-perms" "mariadb-verify-server" "mariadb-ondemand" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
       labels:
 {{ tuple $envAll "mariadb-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
@@ -54,7 +60,7 @@ spec:
         {{ .Values.labels.utility.node_selector_key }}: {{ .Values.labels.utility.node_selector_value }}
       initContainers:
         - name: ondemand-perms
-          image: ${MARIADB_IMAGE_NAME}
+          image: ${MYSQLCLIENT_UTILTIY_IMAGE_NAME}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "ondemand_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
@@ -67,14 +73,30 @@ spec:
               name: pod-tmp
             - mountPath: ${MARIADB_BACKUP_BASE_PATH}
               name: mariadb-backup-dir
+        - name: verify-perms
+          image: ${MYSQLCLIENT_UTILTIY_IMAGE_NAME}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "verify_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - chown
+            - -R
+            - "65534:65534"
+            - /var/lib/mysql
+          volumeMounts:
+            - mountPath: /tmp
+              name: pod-tmp
+            - mountPath: /var/lib/mysql
+              name: mysql-data
       containers:
         - name: mariadb-ondemand
-          image: ${MARIADB_IMAGE_NAME}
+          image: ${MYSQLCLIENT_UTILTIY_IMAGE_NAME}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "mariadb_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          command:
+          command: ["/bin/sh"]
-            - /bin/sleep
+          args:
-            - "1000000"
+            - -c
+            - >-
+                /bin/sleep "1000000"
           env:
             - name: MARIADB_BACKUP_BASE_DIR
               valueFrom:
@@ -216,9 +238,49 @@ if $TLS_ENABLED; then
 {{- if .Values.pod.mounts.mariadb_ondemand.container.mariadb_ondemand.volumeMounts }}
 {{ .Values.pod.mounts.mariadb_ondemand.container.mariadb_ondemand.volumeMounts | toYaml | indent 12 }}
 {{- end }}
+        - name: mariadb-verify-server
+          image: ${MARIADB_IMAGE_NAME}
+{{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.mariadb_verify_server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: MYSQL_HISTFILE
+              value: /dev/null
+          command:
+            - /tmp/start_verification_server.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: var-run
+              mountPath: /var/run/mysqld
+            - name: mycnfd
+              mountPath: /etc/mysql/conf.d
+            - name: mariadb-etc
+              mountPath: /etc/mysql/my.cnf
+              subPath: my.cnf
+              readOnly: true
+            - name: mariadb-secrets
+              mountPath: /etc/mysql/admin_user.cnf
+              subPath: admin_user.cnf
+              readOnly: true
+            - name: mysql-data
+              mountPath: /var/lib/mysql
+            - name: mariadb-bin
+              mountPath: /tmp/start_verification_server.sh
+              subPath: start_verification_server.sh
+              readOnly: true
       volumes:
         - name: pod-tmp
           emptyDir: {}
+        - name: mycnfd
+          emptyDir: {}
+        - name: var-run
+          emptyDir: {}
+        - name: mariadb-etc
+          configMap:
+            name: mariadb-etc
+            defaultMode: 0444
+        - name: mysql-data
+          emptyDir: {}
         - name: mariadb-secrets
           secret:
             secretName: mariadb-secrets
@@ -268,9 +330,49 @@ else
 {{- if .Values.pod.mounts.mariadb_ondemand.container.mariadb_ondemand.volumeMounts }}
 {{ .Values.pod.mounts.mariadb_ondemand.container.mariadb_ondemand.volumeMounts | toYaml | indent 12 }}
 {{- end }}
+        - name: mariadb-verify-server
+          image: ${MARIADB_IMAGE_NAME}
+{{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.mariadb_verify_server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: MYSQL_HISTFILE
+              value: /dev/null
+          command:
+            - /tmp/start_verification_server.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: var-run
+              mountPath: /var/run/mysqld
+            - name: mycnfd
+              mountPath: /etc/mysql/conf.d
+            - name: mariadb-etc
+              mountPath: /etc/mysql/my.cnf
+              subPath: my.cnf
+              readOnly: true
+            - name: mariadb-secrets
+              mountPath: /etc/mysql/admin_user.cnf
+              subPath: admin_user.cnf
+              readOnly: true
+            - name: mysql-data
+              mountPath: /var/lib/mysql
+            - name: mariadb-bin
+              mountPath: /tmp/start_verification_server.sh
+              subPath: start_verification_server.sh
+              readOnly: true
       volumes:
         - name: pod-tmp
           emptyDir: {}
+        - name: mycnfd
+          emptyDir: {}
+        - name: var-run
+          emptyDir: {}
+        - name: mariadb-etc
+          configMap:
+            name: mariadb-etc
+            defaultMode: 0444
+        - name: mysql-data
+          emptyDir: {}
         - name: mariadb-secrets
           secret:
             secretName: mariadb-secrets

charts/mysqlclient-utility/values.yaml

@@ -19,6 +19,7 @@ release_group: null
 
 images:
   tags:
+    mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal
     mysqlclient_utility: 'quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic'
     image_repo_sync: docker.io/docker:18.09.02
   pull_policy: IfNotPresent
@@ -49,10 +50,17 @@ pod:
         ondemand_perms:
           runAsUser: 0
           readOnlyRootFilesystem: true
+        verify_perms:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
         mariadb_ondemand:
           runAsUser: 65534
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
+        mariadb_verify_server:
+          runAsUser: 65534
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
   mounts:
     mysqlclient:
       container:
@@ -83,6 +91,8 @@ pod:
         default: kubernetes.io/hostname
   mandatory_access_control:
     type: apparmor
+    mariadb-verify-server:
+      mariadb-verify-server: runtime/default
     mysqlclient-utility:
       mysqlclient-utility: runtime/default
     mariadb-ondemand:
@@ -97,6 +107,13 @@ pod:
       limits:
         memory: "250Mi"
         cpu: "500m"
+    server:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
     jobs:
       bootstrap:
         limits:

docs/source/conf.py

@@ -41,7 +41,7 @@ release = u'0.1.0'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

kube_utility_container/tests/utility/postgresql/test_postgresql_utility_container.py

@@ -16,6 +16,8 @@ import unittest
 
 from kube_utility_container.tests.utility.base import TestBase
 
+import warnings
+
 class TestPostgresqlUtilityContainer(TestBase):
     @classmethod
     def setUpClass(cls):
@@ -39,6 +41,7 @@ class TestPostgresqlUtilityContainer(TestBase):
 
     def test_verify_postgresql_utility_pod_logs(self):
         """To verify postgresql-utility pod logs"""
+        warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning)
         date_1 = (self.client.exec_cmd(
             self.deployment_name,
             ['date', '+%Y-%m-%d %H'])).replace('\n','')

requirements-frozen.txt

@@ -16,8 +16,8 @@ idna==2.10
 importlib-metadata==3.3.0
 importlib-resources==3.3.0
 iso8601==0.1.13
-kubeconfig==1.0.1
+kubeconfig==1.1.1
-kubernetes==10.0.1
+kubernetes==23.6.0
 linecache2==1.0.0
 monotonic==1.5
 msgpack==1.0.1
@@ -42,13 +42,13 @@ python-dateutil==2.8.1
 python-mimeparse==1.6.0
 python-subunit==1.4.0
 pytz==2020.4
-PyYAML==5.3.1
+PyYAML==5.4.1
 requests==2.25.0
 requests-oauthlib==1.3.0
 rfc3986==1.4.0
 rsa==4.6
 six==1.15.0
-stestr==3.1.0
+stestr==3.2.1
 stevedore==3.3.0
 testtools==2.4.0
 traceback2==1.4.0

requirements.txt

@@ -4,9 +4,9 @@
 
 # When modifying this file `tox -e freeze-req` must be run to regenerate the requirements-frozen.txt.
 coverage==4.5.1
-kubeconfig==1.0.1
+kubeconfig==1.1.1
-kubernetes==10.0.1
+kubernetes==23.6.0
 oslo.config==6.7.0 # Apache-2.0
 oslo.log==3.40.1 # Apache-2.0
 pbr==3.1.1
-stestr==3.1.0 # Apache-2.0
+stestr==3.2.1 # Apache-2.0

tools/deployment/000-install-packages.sh

@@ -4,6 +4,8 @@ set -xe
 cd ${INSTALL_PATH}
 
 # Clone dependencies
+rm -rf openstack-helm-infra
+rm -rf openstack-helm
 git clone https://opendev.org/openstack/openstack-helm-infra.git
 git clone https://opendev.org/openstack/openstack-helm.git
 # Install Packages

tools/deployment/002-build-helm-toolkit.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+CURRENT_DIR="$(pwd)"
+: "${PORTHOLE_PATH:="../porthole"}"
+
+cd "${PORTHOLE_PATH}" || exit
+sudo echo 127.0.0.1 localhost /etc/hosts
+
+BUILD_DIR=$(mktemp -d)
+HELM=${BUILD_DIR}/helm
+HELM_PIDFILE=${CURRENT_DIR}/.helm-pid
+
+rm -rf build
+rm -f charts/*.tgz
+rm -f charts/*/requirements.lock
+rm -rf charts/*/charts
+
+./tools/helm_install.sh ${HELM}
+./tools/helm_tk.sh ${HELM} ${HELM_PIDFILE}
+
+

tools/deployment/002-deploy-k8s.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-CURRENT_DIR="$(pwd)"
-: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
-
-cd "${OSH_INFRA_PATH}"
-bash -c "./tools/deployment/common/005-deploy-k8s.sh"

tools/deployment/003-deploy-k8s.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+CURRENT_DIR="$(pwd)"
+: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
+
+./helm serve
+curl -i http://localhost:8879/charts/
+
+cd "${OSH_INFRA_PATH}"
+bash -c "./tools/deployment/common/005-deploy-k8s.sh"
+
+kubectl create namespace utility
+
+
+curl -i http://localhost:8879/charts/

tools/gate/deploy.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+./tools/deployment/000-install-packages.sh
+./tools/deployment/002-build-helm-toolkit.sh
+./tools/deployment/003-deploy-k8s.sh
+./tools/deployment/005-calicoctl-utility.sh
+./tools/deployment/010-ceph-utility.sh
+./tools/deployment/020-compute-utility.sh
+./tools/deployment/030-etcdctl-utility.sh
+./tools/deployment/040-mysqlclient-utility.sh
+./tools/deployment/050-openstack-utility.sh
+./tools/deployment/060-postgresql-utility.sh
+sleep 30

tools/gate/playbooks/osh-infra-upgrade-host.yaml

@@ -0,0 +1,55 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: False
+  become: yes
+  roles:
+    - deploy-python
+  tags:
+    - deploy-python
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  become: yes
+  roles:
+    - upgrade-host
+    - start-zuul-console
+    - disable-local-nameserver
+    - setup-firewall
+  tags:
+    - upgrade-host
+    - start-zuul-console
+    - disable-local-nameserver
+    - setup-firewall
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: False
+  become: yes
+  roles:
+    - deploy-apparmor
+  tags:
+    - deploy-apparmor
+...

tools/gate/playbooks/roles/deploy-apparmor/tasks/main.yaml

@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- block:
+    - name: ensuring AppArmor is deployed on host
+      when: ansible_distribution == 'Ubuntu'
+      include_role:
+        name: deploy-package
+        tasks_from: dist
+      vars:
+        packages:
+          deb:
+            - apparmor
+
+    - name: "Enable AppArmor"
+      when: ansible_distribution == 'Ubuntu'
+      become: true
+      become_user: root
+      shell: |-
+              set -xe
+              systemctl enable apparmor
+              systemctl start apparmor
+              systemctl status apparmor.service
+      args:
+        executable: /bin/bash
+      ignore_errors: True
+...

tools/gate/playbooks/roles/deploy-docker/defaults/main.yml

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+proxy:
+  http: null
+  https: null
+  noproxy: null
+...

tools/gate/playbooks/roles/deploy-docker/tasks/deploy-ansible-docker-support.yaml

@@ -0,0 +1,70 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: ensuring SELinux is disabled on centos & fedora
+  when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' or ansible_distribution == 'Fedora'
+  become: true
+  become_user: root
+  command: setenforce 0
+  ignore_errors: True
+
+# NOTE(portdirect): See https://ask.openstack.org/en/question/110437/importerror-cannot-import-name-unrewindablebodyerror/
+- name: fix docker removal issue with ansible's docker_container on centos
+  when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux'
+  block:
+    - name: remove requests and urllib3 distro packages to fix docker removal issue with ansible's docker_container on centos
+      include_role:
+        name: deploy-package
+        tasks_from: dist
+      vars:
+        state: absent
+        packages:
+          rpm:
+            - python-urllib3
+            - python-requests
+    - name: restore requests and urllib3 distro packages to fix docker removal issue with ansible's docker_container on centos
+      include_role:
+        name: deploy-package
+        tasks_from: dist
+      vars:
+        state: present
+        packages:
+          rpm:
+            - python-urllib3
+            - python-requests
+
+- name: install additional packages
+  include_role:
+    name: deploy-package
+    tasks_from: dist
+  vars:
+    state: present
+    packages:
+      deb:
+        - conntrack
+        - bc
+        - nmap
+        - ethtool
+      rpm:
+        - conntrack-tools
+        - bc
+        - nmap
+
+- name: Ensure docker python packages deployed
+  include_role:
+    name: deploy-package
+    tasks_from: pip
+  vars:
+    packages:
+      - docker
+...

tools/gate/playbooks/roles/deploy-docker/tasks/main.yaml

@@ -0,0 +1,80 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: setting default limit memlock
+  shell: |
+    set -xe;
+    echo "DefaultLimitMEMLOCK=16777216" | sudo tee -a /etc/systemd/system.conf
+    sudo systemctl daemon-reexec
+    sudo systemctl daemon-reload
+
+- name: check if docker deploy is needed
+  raw: which docker
+  register: need_docker
+  ignore_errors: True
+
+- name: centos | moving systemd unit into place
+  when: ( ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' ) and ( need_docker is failed )
+  template:
+    src: centos-docker.service.j2
+    dest: /etc/systemd/system/docker.service
+    mode: 416
+
+- name: fedora | moving systemd unit into place
+  when: ( ansible_distribution == 'Fedora' ) and ( need_docker is failed )
+  template:
+    src: fedora-docker.service.j2
+    dest: /etc/systemd/system/docker.service
+    mode: 416
+
+- name: ubuntu | moving systemd unit into place
+  when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' ) and ( need_docker is failed )
+  template:
+    src: ubuntu-docker.service.j2
+    dest: /etc/systemd/system/docker.service
+    mode: 416
+
+# NOTE: (lamt) Setting up the proxy before installing docker
+- name: ensure docker.service.d directory exists
+  when: proxy.http
+  file:
+    path: /etc/systemd/system/docker.service.d
+    state: directory
+
+- name: proxy | moving proxy systemd unit into place
+  when: proxy.http
+  template:
+    src: http-proxy.conf.j2
+    dest: /etc/systemd/system/docker.service.d/http-proxy.conf
+    mode: 416
+
+- name: deploy docker packages
+  when: need_docker is failed
+  include_role:
+    name: deploy-package
+    tasks_from: dist
+  vars:
+    packages:
+      deb:
+        - docker.io
+      rpm:
+        - docker
+
+- name: restarting docker
+  systemd:
+    state: restarted
+    daemon_reload: yes
+    name: docker
+
+- include: deploy-ansible-docker-support.yaml
+...

tools/gate/playbooks/roles/deploy-docker/templates/centos-docker.service.j2

@@ -0,0 +1,35 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=http://docs.docker.com
+After=network.target
+
+[Service]
+Type=notify
+NotifyAccess=all
+Environment=GOTRACEBACK=crash
+Environment=DOCKER_HTTP_HOST_COMPAT=1
+Environment=PATH=/usr/libexec/docker:/usr/bin:/usr/sbin
+ExecStart=/usr/bin/dockerd-current \
+          --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current \
+          --default-runtime=docker-runc \
+          --exec-opt native.cgroupdriver=systemd \
+          --userland-proxy-path=/usr/libexec/docker/docker-proxy-current \
+          --seccomp-profile=/etc/docker/seccomp.json \
+          --graph=/var/lib/docker \
+          --storage-driver=overlay2 \
+          --log-driver=json-file \
+          --iptables=false
+# NOTE(portdirect): fix mount propagation for CentOS, this is done post start,
+# as docker seems to reset this.
+ExecStartPost=/usr/bin/mount --make-rshared /
+ExecReload=/bin/kill -s HUP $MAINPID
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=0
+Restart=on-abnormal
+MountFlags=share
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

tools/gate/playbooks/roles/deploy-docker/templates/fedora-docker.service.j2

@@ -0,0 +1,34 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=http://docs.docker.com
+After=network.target docker-containerd.service
+Requires=docker-containerd.service
+
+[Service]
+Type=notify
+Environment=GOTRACEBACK=crash
+ExecStart=/usr/bin/dockerd-current \
+          --add-runtime oci=/usr/libexec/docker/docker-runc-current \
+          --default-runtime=oci \
+          --containerd /run/containerd.sock \
+          --exec-opt native.cgroupdriver=systemd \
+          --userland-proxy-path=/usr/libexec/docker/docker-proxy-current \
+          --init-path=/usr/libexec/docker/docker-init-current \
+          --seccomp-profile=/etc/docker/seccomp.json \
+          --graph=/var/lib/docker \
+          --storage-driver=overlay2 \
+          --log-driver=json-file \
+          --iptables=false
+# NOTE(portdirect): fix mount propagation for Fedora, this is done post start,
+# as docker seems to reset this.
+ExecStartPost=/usr/bin/mount --make-rshared /
+ExecReload=/bin/kill -s HUP $MAINPID
+TasksMax=8192
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=0
+Restart=on-abnormal
+
+[Install]
+WantedBy=multi-user.target

tools/gate/playbooks/roles/deploy-docker/templates/http-proxy.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+Environment="HTTP_PROXY={{ proxy.http }}"
+Environment="HTTPS_PROXY={{ proxy.https }}"
+Environment="NO_PROXY={{ proxy.noproxy }}"

tools/gate/playbooks/roles/deploy-docker/templates/ubuntu-docker.service.j2

@@ -0,0 +1,30 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network.target docker.socket firewalld.service
+Requires=docker.socket
+
+[Service]
+Type=notify
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+EnvironmentFile=-/etc/default/docker
+ExecStart=/usr/bin/dockerd --iptables=false -H fd:// $DOCKER_OPTS
+ExecReload=/bin/kill -s HUP $MAINPID
+LimitNOFILE=1048576
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+# Uncomment TasksMax if your systemd version supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+TimeoutStartSec=0
+# set delegate yes so that systemd does not reset the cgroups of docker containers
+Delegate=yes
+# kill only the docker process, not all processes in the cgroup
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

tools/gate/playbooks/roles/deploy-jq/tasks/main.yaml

@@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- block:
+    - name: ensuring jq is deployed on host
+      when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Fedora'
+      include_role:
+        name: deploy-package
+        tasks_from: dist
+      vars:
+        packages:
+          deb:
+            - jq
+          rpm:
+            - jq
+
+    - name: installing jq 1.5 binary for centos
+      become: true
+      become_user: root
+      when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux'
+      get_url:
+        url: https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
+        dest: /usr/bin/jq
+        mode: 365
+        force: yes
+...

tools/gate/playbooks/roles/deploy-package/defaults/main.yml

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+proxy:
+  http: null
+  https: null
+  noproxy: null
+...

tools/gate/playbooks/roles/deploy-package/tasks/dist.yaml

@@ -0,0 +1,46 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: managing distro packages for ubuntu
+  become: true
+  become_user: root
+  when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
+  vars:
+    state: present
+  apt:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages.deb }}"
+
+- name: managing distro packages for centos
+  become: true
+  become_user: root
+  when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux'
+  vars:
+    state: present
+  yum:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages.rpm }}"
+
+- name: managing distro packages for fedora
+  become: true
+  become_user: root
+  when: ansible_distribution == 'Fedora'
+  vars:
+    state: present
+  dnf:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages.rpm }}"
+...

tools/gate/playbooks/roles/deploy-package/tasks/pip.yaml

@@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: managing pip packages
+  become: true
+  become_user: root
+  environment:
+    http_proxy: "{{ proxy.http }}"
+    https_proxy: "{{ proxy.https }}"
+    no_proxy: "{{ proxy.noproxy }}"
+  vars:
+    state: present
+  pip:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages }}"
+...

tools/gate/playbooks/roles/deploy-python-pip/defaults/main.yml

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+proxy:
+  http: null
+  https: null
+  noproxy: null
+...

tools/gate/playbooks/roles/deploy-python-pip/tasks/main.yaml

@@ -0,0 +1,55 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: check if pip installed
+  command: pip3 --version
+  register: pip_version_output
+  ignore_errors: yes
+  changed_when: false
+
+- name: ensuring python pip package is present for ubuntu
+  when: ( pip_version_output is failed ) and ( ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' )
+  apt:
+    name: python3-pip
+    state: present
+
+- name: ensuring python pip package is present for centos
+  when: ( pip_version_output is failed ) and ( ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' )
+  block:
+    - name: ensuring epel-release package is present for centos as python3-pip is in the epel repo
+      yum:
+        name: epel-release
+        state: present
+    - name: ensuring python pip package is present for centos
+      yum:
+        name: python3-pip
+        state: present
+
+- name: ensuring python pip package is present for fedora via the python3-pip rpm
+  when: ( pip_version_output is failed ) and ( ansible_distribution == 'Fedora' )
+  dnf:
+    name: python3-pip
+    state: present
+
+- name: ensuring pip is the latest version
+  become: true
+  become_user: root
+  environment:
+    http_proxy: "{{ proxy.http }}"
+    https_proxy: "{{ proxy.https }}"
+    no_proxy: "{{ proxy.noproxy }}"
+  pip:
+    name: pip
+    state: latest
+    executable: pip3
+...

tools/gate/playbooks/roles/deploy-python/tasks/main.yaml

@@ -0,0 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: ensuring python3 is present on all hosts
+  raw: test -e /usr/bin/python3 || (sudo apt -y update && sudo apt install -y python3-minimal) || (sudo yum install -y python3) || (sudo dnf install -y python3)
+...

tools/gate/playbooks/roles/disable-local-nameserver/tasks/main.yaml

@@ -0,0 +1,59 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE(portdirect): We disable the local nameserver as it interferes with the
+# k8s dns-service and other local resolvers used for development use.
+# See the following for the original config:
+# * https://github.com/openstack/project-config/blob/0332c33dd134033e0620645c252f82b77e4c16f5/nodepool/elements/nodepool-base/finalise.d/89-unbound
+
+---
+- name: Disable local nameserver and systemd-resolved service
+  when: ansible_distribution == 'Ubuntu'
+  block:
+    - name: update rc.local
+      blockinfile:
+        path: /etc/rc.local
+        mode: 365
+        block: |
+          #!/bin/bash
+          set -o xtrace
+          # Some providers inject dynamic network config statically. Work around this
+          # for DNS nameservers. This is expected to fail on some nodes so remove -e.
+          set +e
+          sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-*
+          sed -i -e 's/^NETCONFIG_DNS_POLICY=.*/NETCONFIG_DNS_POLICY=""/g' /etc/sysconfig/network/config
+          set -e
+          echo 'nameserver 208.67.222.222' > /etc/resolv.conf
+          echo 'nameserver 8.8.8.8' >> /etc/resolv.conf
+          exit 0
+    - name: write resolv.conf
+      blockinfile:
+        path: /etc/resolv.conf
+        mode: 644
+        block: |
+          nameserver 208.67.222.222
+          nameserver 8.8.8.8
+    - name: stop unbound service
+      systemd:
+        state: stopped
+        enabled: no
+        masked: yes
+        daemon_reload: yes
+        name: unbound
+    - name: stop systemd-resolved service
+      systemd:
+        state: stopped
+        enabled: no
+        masked: yes
+        daemon_reload: yes
+        name: systemd-resolved
+...

tools/gate/playbooks/roles/setup-firewall/tasks/main.yaml

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE(portdirect): This needs refinement but drops the firewall on zuul nodes
+---
+- name: deploy iptables packages
+  include_role:
+    name: deploy-package
+    tasks_from: dist
+  vars:
+    packages:
+      deb:
+        - iptables
+      rpm:
+        - iptables
+- command: iptables -S
+- command: iptables -F
+- command: iptables -P INPUT ACCEPT
+- command: iptables -S
+...

tools/gate/playbooks/roles/upgrade-host/defaults/main.yml

@@ -0,0 +1,15 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ubuntu_kernel_hwe: false
+...

tools/gate/playbooks/roles/upgrade-host/tasks/main.yaml

@@ -0,0 +1,44 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Upgrade to HWE kernel on Ubuntu Hosts
+  when:
+    - ansible_distribution == 'Ubuntu'
+    - ubuntu_kernel_hwe == true
+  block:
+    - name: Deploy HWE kernel on Ubuntu Hosts
+      include_role:
+        name: deploy-package
+        tasks_from: dist
+      vars:
+        packages:
+          deb:
+            - linux-generic-hwe-16.04
+    - name: Reboot Host following kernel upgrade
+      shell: sleep 2 && reboot
+      become: yes
+      async: 30
+      poll: 0
+      ignore_errors: true
+      args:
+        executable: /bin/bash
+    - name: Wait for hosts to come up following reboot
+      wait_for:
+        host: '{{ hostvars[item].ansible_host }}'
+        port: 22
+        state: started
+        delay: 60
+        timeout: 240
+      with_items: '{{ play_hosts }}'
+      connection: local
+...

tools/gate/playbooks/vars.yaml

@@ -15,3 +15,5 @@ docker_daemon:
   registry-mirrors:
     - "http://{{ zuul_site_mirror_fqdn }}:8082/"
   storage-driver: overlay2
+ansible_python_interpreter: python3
+...

tools/helm_install.sh

@@ -24,19 +24,19 @@ function install_helm_binary {
   if [[ -z "${HELM}" ]]
   then
     echo "No Helm binary target location."
-    exit -1
+    exit 1
   fi
 
   if [[ -w "$(dirname ${HELM})" ]]
   then
     TMP_DIR=${BUILD_DIR:-$(mktemp -d)}
     curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}"
-    cd ${TMP_DIR}
+    cd ${TMP_DIR} || exit
     tar -xvzf helm.tar.gz
     cp "${TMP_DIR}/linux-amd64/helm" "${HELM}"
   else
     echo "Cannot write to ${HELM}"
-    exit -1
+    exit 1
   fi
 }
 

tools/helm_tk.sh

@@ -22,7 +22,7 @@ HELM=${1}
 HELM_PIDFILE=${2}
 SERVE_DIR=$(mktemp -d)
 
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"b1a247e7f54ab12d830ab74f7634457b4e43f3ef"}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"5c4056ad341afcc577e63902b6ddbfb222d757e1"}
 
 ${HELM} init --client-only --skip-refresh --stable-repo-url "https://charts.helm.sh/stable"
 
@@ -69,11 +69,11 @@ fi
 
 {
     cd "${SERVE_DIR}"
+    rm -rf openstack-helm-infra
     git clone https://git.openstack.org/openstack/openstack-helm-infra.git || true
     cd openstack-helm-infra
     git reset --hard "${HTK_STABLE_COMMIT}"
-
     make helm-toolkit
 }
 
-rm -rf "${SERVE_DIR}"
+# rm -rf "${SERVE_DIR}"

tools/run_avt.sh

@@ -20,7 +20,7 @@ PLUGINS=kube_utility_container
 export KUBECONFIG=${KUBECONFIG:-~/.kube/config}
 
 function setup_venv() {
-    sudo  apt-get install libffi-dev libssl-dev -y
+    sudo  apt-get install libffi-dev libssl-dev python3-dev python3-setuptools python3-venv gcc make build-essential automake autoconf -y
     python3 -m venv ${VENV}
     if [[ -f ${VENV}/bin/activate ]] ;then
       source $VENV/bin/activate

zuul.d/base.yaml

@@ -55,6 +55,12 @@
       - name: primary
         label: ubuntu-bionic
 
+- nodeset:
+    name: airship-porthole-focal-single-node
+    nodes:
+      - name: primary
+        label: ubuntu-focal
+
 - job:
     name: airship-porthole-images
     abstract: true
@@ -85,23 +91,16 @@
     description: |
        Deploys all UC's and Executes unit and feature tests.
     run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    nodeset: airship-porthole-single-node
+    nodeset: airship-porthole-focal-single-node
     timeout: 7200
+    pre-run:
+       - tools/gate/playbooks/osh-infra-upgrade-host.yaml
     post-run:
        - tools/gate/playbooks/make-unit-tests.yaml
        - tools/gate/playbooks/make-feature-tests.yaml
     vars:
       gate_scripts:
-         - ./tools/deployment/000-install-packages.sh
+         - ./tools/gate/deploy.sh
-         - ./tools/deployment/001-setup-apparmor-profiles.sh
-         - ./tools/deployment/002-deploy-k8s.sh
-         - ./tools/deployment/005-calicoctl-utility.sh
-         - ./tools/deployment/010-ceph-utility.sh
-         - ./tools/deployment/020-compute-utility.sh
-         - ./tools/deployment/030-etcdctl-utility.sh
-         - ./tools/deployment/040-mysqlclient-utility.sh
-         - ./tools/deployment/050-openstack-utility.sh
-         - ./tools/deployment/060-postgresql-utility.sh
       args:
         chdir: "{{ zuul.project.src_dir }}"