From 91ae8e62dee37d4ded6e1066d7247090244c7e77 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jeblair@redhat.com>
Date: Thu, 17 Oct 2019 15:21:08 -0700
Subject: [PATCH] Test buildset registry with k8s and docker
This adds a test job for the buildset registry with k8s with the
docker runtime.
Change-Id: I0ec1d3d5980424f85f5e7619b9dec96c8f0f70f1
---
.../templates/registry.yaml.j2 | 2 +-
.../buildset-registry-k8s-docker-post.yaml | 8 ++
.../buildset-registry-k8s-docker.yaml | 26 ++++++
.../registry/buildset-registry-pre.yaml | 39 +++++++++
.../registry/buildset-registry.yaml | 32 +++++++
test-playbooks/registry/docker/Dockerfile | 2 +-
.../registry/test-registry-post.yaml | 1 +
zuul-tests.d/container-roles-jobs.yaml | 83 +++++++++++++++++++
8 files changed, 191 insertions(+), 2 deletions(-)
create mode 100644 test-playbooks/registry/buildset-registry-k8s-docker-post.yaml
create mode 100644 test-playbooks/registry/buildset-registry-k8s-docker.yaml
create mode 100644 test-playbooks/registry/buildset-registry-pre.yaml
create mode 100644 test-playbooks/registry/buildset-registry.yaml
diff --git a/roles/run-buildset-registry/templates/registry.yaml.j2 b/roles/run-buildset-registry/templates/registry.yaml.j2
index 7901b727f..81667eba5 100644
--- a/roles/run-buildset-registry/templates/registry.yaml.j2
+++ b/roles/run-buildset-registry/templates/registry.yaml.j2
@@ -1,7 +1,7 @@
registry:
address: '::'
port: 5000
- public-url: 'https://{{ ansible_host | ipwrap }}:5000'
+ public-url: 'https://{{ ansible_host | ipwrap }}:{{ buildset_registry_port }}'
tls-cert: /tls/cert.pem
tls-key: /tls/cert.key
secret: "{{ registry_secret }}"
diff --git a/test-playbooks/registry/buildset-registry-k8s-docker-post.yaml b/test-playbooks/registry/buildset-registry-k8s-docker-post.yaml
new file mode 100644
index 000000000..371fab2a7
--- /dev/null
+++ b/test-playbooks/registry/buildset-registry-k8s-docker-post.yaml
@@ -0,0 +1,8 @@
+- hosts: all
+ tasks:
+ - name: Describe dockertest pod
+ command: kubectl describe pod/dockertest
+ ignore_errors: true
+ - name: Describe upstream-dockertest pod
+ command: kubectl describe pod/upstream-dockertest
+ ignore_errors: true
diff --git a/test-playbooks/registry/buildset-registry-k8s-docker.yaml b/test-playbooks/registry/buildset-registry-k8s-docker.yaml
new file mode 100644
index 000000000..a345c7678
--- /dev/null
+++ b/test-playbooks/registry/buildset-registry-k8s-docker.yaml
@@ -0,0 +1,26 @@
+- hosts: all
+ roles:
+ - role: clear-firewall
+ - role: install-kubernetes
+ - role: use-buildset-registry
+ buildset_registry_docker_user: root
+ tasks:
+ - name: Wait for cluster to come up
+ command: kubectl cluster-info
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 30
+ - name: Run a local test pod
+ command: kubectl run --generator=run-pod/v1 --image=zuul/docker-testimage dockertest
+ - name: Wait for the pod to be ready
+ command: kubectl wait --for=condition=Ready pod/dockertest --timeout=60s
+ - name: Check the output of the pod
+ shell: "kubectl logs pod/dockertest | grep 'Zuul container test'"
+
+ - name: Run a remote test pod
+ command: kubectl run --generator=run-pod/v1 --image=debian:testing upstream-dockertest --command -- /bin/bash -c 'echo Upstream; sleep infinity'
+ - name: Wait for the pod to be ready
+ command: kubectl wait --for=condition=Ready pod/upstream-dockertest --timeout=60s
+ - name: Check the output of the pod
+ shell: "kubectl logs pod/upstream-dockertest | grep 'Upstream'"
diff --git a/test-playbooks/registry/buildset-registry-pre.yaml b/test-playbooks/registry/buildset-registry-pre.yaml
new file mode 100644
index 000000000..455d9f800
--- /dev/null
+++ b/test-playbooks/registry/buildset-registry-pre.yaml
@@ -0,0 +1,39 @@
+- hosts: all
+ tasks:
+ - name: Add project atomic PPA
+ become: true
+ apt_repository:
+ repo: ppa:projectatomic/ppa
+ - name: Install packages
+ become: true
+ package:
+ name:
+ - skopeo
+ state: present
+
+ # This happens in the pre-playbook of the
+ # opendev-buildset-registry job. Since we're testing these roles
+ # in opendev, and we're running a second buildset registry, we
+ # need to do the same here for it.
+ - name: Open the IPv4 port for the second buildset registry
+ become: true
+ iptables:
+ action: insert
+ chain: openstack-INPUT
+ destination_port: '9000'
+ jump: ACCEPT
+ match: tcp
+ ctstate: NEW
+ protocol: tcp
+ ip_version: ipv4
+ - name: Open the IPv6 port for the second buildset registry
+ become: true
+ iptables:
+ action: insert
+ chain: openstack-INPUT
+ destination_port: '9000'
+ jump: ACCEPT
+ match: tcp
+ ctstate: NEW
+ protocol: tcp
+ ip_version: ipv6
diff --git a/test-playbooks/registry/buildset-registry.yaml b/test-playbooks/registry/buildset-registry.yaml
new file mode 100644
index 000000000..a0e6e4507
--- /dev/null
+++ b/test-playbooks/registry/buildset-registry.yaml
@@ -0,0 +1,32 @@
+# This job inherits from a buildset-registry job, so it should already
+# be running locally and have any speculative images loaded into it.
+- hosts: all
+ tasks:
+ - name: Load real buildset registry connection info
+ set_fact:
+ real_buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
+ # This should now use the speculative image, because we've already
+ # run use-buildset-registry.
+ - name: Run the fake buildset registry
+ include_role:
+ role: run-buildset-registry
+ vars:
+ buildset_registry_root: "{{ ansible_user_dir }}/fake_buildset_registry"
+ buildset_registry_port: 9000
+ # Leave that zuul return so that dependent jobs use the fake one
+ - name: Load fake buildset registry connection info
+ set_fact:
+ fake_buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
+ - name: Build a test image
+ command: "docker build . -t zuul/testimage:latest"
+ args:
+ chdir: "{{ zuul.project.src_dir }}/test-playbooks/registry/docker"
+ - name: Push test image into fake buildset registry
+ command: "skopeo copy --dest-tls-verify=false --dest-creds {{ buildset_registry.username }}:{{ buildset_registry.password }} docker-daemon:zuul/testimage:latest docker://localhost:9000/zuul/docker-testimage:latest"
+ - name: Push test image into fake buildset registry
+ command: "skopeo copy --dest-tls-verify=false --dest-creds {{ buildset_registry.username }}:{{ buildset_registry.password }} docker-daemon:zuul/testimage:latest docker://localhost:9000/quay.io/zuul/quay-testimage:latest"
+ - name: Pause the job
+ zuul_return:
+ data:
+ zuul:
+ pause: true
diff --git a/test-playbooks/registry/docker/Dockerfile b/test-playbooks/registry/docker/Dockerfile
index fe50986d3..609bf905a 100644
--- a/test-playbooks/registry/docker/Dockerfile
+++ b/test-playbooks/registry/docker/Dockerfile
@@ -1,2 +1,2 @@
FROM debian:testing
-RUN touch /tmp/foo
+CMD echo "Zuul container test"; sleep infinity
diff --git a/test-playbooks/registry/test-registry-post.yaml b/test-playbooks/registry/test-registry-post.yaml
index f4ffe0381..108993925 100644
--- a/test-playbooks/registry/test-registry-post.yaml
+++ b/test-playbooks/registry/test-registry-post.yaml
@@ -15,6 +15,7 @@
shell: "docker logs {{ item }} &> {{ ansible_user_dir }}/zuul-output/logs/docker/{{ item }}.txt"
args:
executable: /bin/bash
+ ignore_errors: true
- name: Open container logs permissions
file:
diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml
index 694f5451f..e6fac892d 100644
--- a/zuul-tests.d/container-roles-jobs.yaml
+++ b/zuul-tests.d/container-roles-jobs.yaml
@@ -26,6 +26,87 @@
- name: builder
label: ubuntu-bionic
+- job:
+ name: zuul-jobs-test-registry-buildset-registry
+ parent: opendev-buildset-registry
+ description: |
+ Run a buildset registry for the test-registry jobs
+
+ This runs two registries: a real buildset registry so that we
+ can receive speculative zuul-registry images, and a fake
+ buildset registry (running the speculative or latest
+ zuul-registry) that is used to test using the buildset registry
+ role.
+
+ It is not meant to be used directly but rather run on changes
+ to roles in the zuul-jobs repo.
+ files:
+ - roles/pull-from-intermediate-registry/.*
+ - roles/push-to-intermediate-registry/.*
+ - roles/install-docker/.*
+ - roles/install-kubernetes/.*
+ - roles/build-docker-image/.*
+ - roles/run-buildset-registry/.*
+ - roles/use-buildset-registry/.*
+ - test-playbooks/registry/.*
+ pre-run: test-playbooks/registry/buildset-registry-pre.yaml
+ run: test-playbooks/registry/buildset-registry.yaml
+ post-run: test-playbooks/registry/test-registry-post.yaml
+
+- job:
+ name: zuul-jobs-test-registry-buildset-registry-k8s-docker
+ dependencies: zuul-jobs-test-registry-buildset-registry
+ description: |
+ Test a buildset registry with kubernetes and docker
+
+ It is not meant to be used directly but rather run on changes
+ to roles in the zuul-jobs repo.
+ files:
+ - roles/pull-from-intermediate-registry/.*
+ - roles/push-to-intermediate-registry/.*
+ - roles/install-docker/.*
+ - roles/install-kubernetes/.*
+ - roles/build-docker-image/.*
+ - roles/run-buildset-registry/.*
+ - roles/use-buildset-registry/.*
+ - test-playbooks/registry/.*
+ run: test-playbooks/registry/buildset-registry-k8s-docker.yaml
+ post-run:
+ - test-playbooks/registry/buildset-registry-k8s-docker-post.yaml
+ - test-playbooks/registry/test-registry-post.yaml
+
+- job:
+ name: zuul-jobs-test-install-kubernetes-docker
+ description: |
+ Test the install-kubernetes role with docker
+
+ This job tests changes to the install-kubernetes roles. It
+ is not meant to be used directly but rather run on changes to
+ roles in the zuul-jobs repo.
+ files:
+ - roles/install-docker/.*
+ - roles/install-kubernetes/.*
+ - test-playbooks/install-kubernetes.yaml
+ run: test-playbooks/install-kubernetes/docker.yaml
+ post-run: test-playbooks/install-kubernetes/post.yaml
+
+- job:
+ name: zuul-jobs-test-install-kubernetes-crio
+ description: |
+ Test the install-kubernetes role with crio-o
+
+ This job tests changes to the install-kubernetes roles. It
+ is not meant to be used directly but rather run on changes to
+ roles in the zuul-jobs repo.
+ nodeset:
+ nodes:
+ - name: intermediate-registry
+ label: ubuntu-bionic
+ - name: executor
+ label: ubuntu-bionic
+ - name: builder
+ label: ubuntu-bionic
+
- job:
name: zuul-jobs-test-install-kubernetes-docker
description: |
@@ -62,6 +143,8 @@
check:
jobs: &id001
- zuul-jobs-test-registry
+ - zuul-jobs-test-registry-buildset-registry
+ - zuul-jobs-test-registry-buildset-registry-k8s-docker
- zuul-jobs-test-install-kubernetes-docker
- zuul-jobs-test-install-kubernetes-crio
gate: