Integration tests update

Change-Id: I9b0efe3f2e6fcb82450197d6349be30e853a55b8

.zuul.yaml

@@ -22,6 +22,8 @@
         - airship-drydock-openstack-tox-cover-jammy
         - airship-drydock-openstack-tox-pep8-jammy
         - airship-drydock-baclient-test
+        - airship-drydock-run-test
+        - airship-drydock-integration-test
         - airship-drydock-chart-build-gate
         - airship-drydock-chart-build-latest-htk
         - airship-drydock-docker-build-gate-ubuntu_jammy
@@ -32,6 +34,8 @@
         - airship-drydock-openstack-tox-cover-jammy
         - airship-drydock-openstack-tox-pep8-jammy
         - airship-drydock-baclient-test
+        - airship-drydock-run-test
+        - airship-drydock-integration-test
         - airship-drydock-chart-build-gate
         - airship-drydock-docker-build-gate-ubuntu_jammy
 
@@ -58,13 +62,16 @@
     description: Runs cover job on jammy
     nodeset: airship-drydock-single-node-jammy
     pre-run: tools/gate/playbooks/install-docker.yaml
-
+    vars:
+      tox_environment:
+        DISTRO: ubuntu_jammy
 
 - job:
     name: airship-drydock-openstack-tox-pep8-jammy
     parent: openstack-tox-pep8
     description: Runs pep8 job on jammy
     nodeset: airship-drydock-single-node-jammy
+    pre-run: tools/gate/playbooks/install-docker.yaml
 
 - job:
     name: airship-drydock-baclient-test
@@ -73,6 +80,29 @@
     run: tools/gate/playbooks/baclient_test.yaml
     timeout: 3600
     nodeset: airship-drydock-single-node-jammy
+    pre-run: tools/gate/playbooks/install-docker.yaml
+
+- job:
+    name: airship-drydock-run-test
+    description: |
+      Run a bootaction drydock client test
+    run: tools/gate/playbooks/run_drydock.yaml
+    timeout: 3600
+    nodeset: airship-drydock-single-node-jammy
+    pre-run: tools/gate/playbooks/install-docker.yaml
+    vars:
+      distro: ubuntu_jammy
+
+- job:
+    name: airship-drydock-integration-test
+    description: |
+      Run a bootaction drydock client test
+    run: tools/gate/playbooks/integration_test.yaml
+    timeout: 3600
+    nodeset: airship-drydock-single-node-jammy
+    pre-run: tools/gate/playbooks/install-docker.yaml
+    vars:
+      distro: ubuntu_jammy
 
 - job:
     name: airship-drydock-chart-build-gate

Makefile

@@ -48,7 +48,7 @@ coverage_test: build_drydock
 
 # Run just unit tests
 unit_tests:
-	tox -re py38 $(TESTS)
+	tox -re py310 $(TESTS)
 
 # Run just DB integration tests
 db_integration_tests:

python/tests/postgres/start_postgres.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
 set -x
 
+env
+echo "DISTRO is set to: $DISTRO"
 DOCKER_REGISTRY=${DOCKER_REGISTRY:-"quay.io"}
 IMAGE_PREFIX=${IMAGE_PREFIX:-"airshipit"}
 IMAGE_NAME=${IMAGE_NAME:-"drydock"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
-DISTRO=${DISTRO:-"ubuntu_focal"}
+DISTRO=${DISTRO:-"ubuntu_jammy"}
 
 
 IMAGE="${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}"

tools/baclient_build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Install golang-go package, and build the baclient library
-set -x
+set -ex
 
 if $(uname -a | grep -q Ubuntu); then
   GOPATH=$1

tools/drydock_image_run.sh

@@ -10,6 +10,11 @@ function start_db {
     then
       sudo docker stop "${PSQL_CONTAINER_NAME}"
     fi
+    if [[ ! -z $(docker ps | grep "${DRYDOCK_CONTAINER_NAME}" ) ]]
+    then
+      sudo docker stop "${DRYDOCK_CONTAINER_NAME}"
+    fi
+
 
     docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust quay.io/airshipit/postgres:14.8
     sleep 15
@@ -49,6 +54,7 @@ function init_db {
 function test_drydock {
     TMPETC=$1
     docker run \
+      --rm \
       -d --name "${DRYDOCK_CONTAINER_NAME}" --net host \
       -v ${TMPETC}:/etc/drydock \
       ${IMAGE}
@@ -78,7 +84,6 @@ function cleanup {
     TMPDIR=$1
     docker stop "${PSQL_CONTAINER_NAME}"
     docker stop "${DRYDOCK_CONTAINER_NAME}"
-    docker rm "${DRYDOCK_CONTAINER_NAME}"
    rm -rf $TMPDIR
 }
 

tools/gate/playbooks/baclient_test.yaml

@@ -14,22 +14,10 @@
 # to minimize Zuul node consumption
 
 - hosts: primary
-
-
-
-  roles:
-    - bindep
-    - ensure-docker
-    - ensure-python
-    - ensure-pip
-    - ensure-tox
-
-
   tasks:
     - name: Execute Go tests for baclient
-      make:
+      shell: |
+        make test_baclient
+      args:
         chdir: "{{ zuul.project.src_dir }}"
-        target: test_baclient
-      become: true
-      register: result
 

tools/gate/playbooks/docker-image-build.yaml

@@ -61,15 +61,6 @@
             executable: pip3
       become: True
 
-    - name: Run images
-      shell: |
-        make run_images
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
-        executable: /bin/bash
-      become: True
-
-
     - name: Make images
       when: not publish
       block:

tools/gate/playbooks/install-docker.yaml

@@ -18,11 +18,22 @@
     - bindep
     - clear-firewall
     - ensure-docker
+    - ensure-python
+    - ensure-pip
+    - ensure-tox
+
   tasks:
+
+    - name: Install tox python module
+      block:
+        - pip:
+            name: tox
+            executable: pip3
+      become: True
+
     - name: Build images
       shell: |
         make images
       args:
         chdir: "{{ zuul.project.src_dir }}"
         executable: /bin/bash
-      become: True

tools/gate/playbooks/integration_test.yaml

@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+# Combine several test tasks into a single playbook
+# to minimize Zuul node consumption
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for running the Drydock Docker image
+      shell: |
+        #/bin/bash
+        set -ex
+        make db_integration_tests
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      become: true

tools/gate/playbooks/run_drydock.yaml

@@ -0,0 +1,26 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+# Combine several test tasks into a single playbook
+# to minimize Zuul node consumption
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for running the Drydock Docker image
+      shell: |
+        #/bin/bash
+        set -ex
+        make run_drydock
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      become: true
+

tox.ini

@@ -5,7 +5,18 @@ minversion = 3.28.0
 
 [testenv]
 setenv = YAMLDIR = {toxinidir}/python/tests/yaml_samples/
-passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY
+passenv =
+  http_proxy
+  HTTP_PROXY
+  https_proxy
+  HTTPS_PROXY
+  no_proxy
+  NO_PROXY
+  DOCKER_REGISTRY
+  IMAGE_NAME
+  IMAGE_PREFIX
+  IMAGE_TAG
+  DISTRO
 deps=
   -r{toxinidir}/python/requirements-frozen.txt
   -r{toxinidir}/python/test-requirements.txt
@@ -47,7 +58,12 @@ commands=
     {toxinidir}/python/tests/unit/{posargs}
 
 [testenv:integration]
-passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO
+passenv =
+  DOCKER_REGISTRY
+  IMAGE_NAME
+  IMAGE_PREFIX
+  IMAGE_TAG
+  DISTRO
 setenv=
   PYTHONWARNING=all
   YAMLDIR={toxinidir}/python/tests/yaml_samples/
@@ -63,12 +79,19 @@ commands=
 
 [testenv:cover]
 usedevelop=True
-passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO
+passenv =
+  DOCKER_REGISTRY
+  IMAGE_NAME
+  IMAGE_PREFIX
+  IMAGE_TAG
+  DISTRO
 setenv=
   YAMLDIR={toxinidir}/python/tests/yaml_samples/
 allowlist_externals=
   sh
+  env
 commands=
+  env
   sh -c {toxinidir}/python/tests/postgres/start_postgres.sh
   py.test \
     --cov=drydock_provisioner \
@@ -99,10 +122,10 @@ commands =
 [testenv:pep8]
 commands =
   flake8 {posargs}
-  bandit -r drydock_provisioner -n 5
+  bandit -r ./python/drydock_provisioner -n 5 --skip B303
 
 [testenv:bandit]
-commands = bandit -r drydock_provisioner -n 5
+commands = bandit -r ./python/drydock_provisioner -n 5 --skip B303
 
 [flake8]
 ignore=E302,H306,H304,W503,E251,E126