diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml new file mode 100644 index 00000000..78d670f6 --- /dev/null +++ b/.zuul.d/jobs.yaml @@ -0,0 +1,39 @@ +- job: + name: ara-integration-base + parent: base + files: + - ara/* + - playbooks/* + - roles/* + - tests/* + - .zuul.d/* + - manage.py + - setup.py + - setup.cfg + - requirements.txt + - test-requirements.txt + vars: + ara_tests_ansible_name: "{{ ansible_user_dir }}/src/github.com/ansible/ansible" + ara_tests_ansible_version: null + ara_api_secure_logging: false + post-run: tests/zuul_post_logs.yaml + +- job: + name: ara-api-database-backends + parent: ara-integration-base + required-projects: + - name: github.com/ansible/ansible + override-checkout: stable-2.9 + pre-run: tests/zuul_pre_multinode_networking.yaml + +- job: + name: ara-api-mysql + parent: ara-api-database-backends + nodeset: ara-database-server-multinode + description: | + Deploys the ARA API server on Ubuntu 18.04 as well as Fedora 30 and + tests it against a central MySQL server installed on CentOS 8. + The job exercises the ara_api Ansible role, the ARA Ansible plugins, the + ARA API clients as well as the API itself. + run: tests/with_mysql.yaml + post-run: tests/zuul_post_with_mysql.yaml diff --git a/.zuul.d/nodesets.yaml b/.zuul.d/nodesets.yaml new file mode 100644 index 00000000..d286eab8 --- /dev/null +++ b/.zuul.d/nodesets.yaml @@ -0,0 +1,19 @@ +# Nodeset used to test instances of ARA API deployed on different operating +# systems against MySQL and PostgreSQL simultaneously. +- nodeset: + name: ara-database-server-multinode + nodes: + - name: database-server + label: centos-8 + - name: ubuntu-bionic + label: ubuntu-bionic + - name: fedora-30 + label: fedora-30 + groups: + - name: ara-database-server + nodes: + - database-server + - name: ara-api-server + nodes: + - ubuntu-bionic + - fedora-30 diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index f00949a5..659087a3 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -5,12 +5,12 @@ - docs-on-readthedocs check: jobs: + - ara-api-mysql - ansible-role-ara-api-fedora-devel - ansible-role-ara-api-fedora-2.9 - ansible-role-ara-api-ubuntu-2.8 - ansible-role-ara-api-ubuntu-2.7 - ansible-role-ara-api-ubuntu-postgresql - - ansible-role-ara-api-ubuntu-mysql - ansible-role-ara-api-fedora-distributed-sqlite - ansible-role-ara-api-fedora-packages: voting: false @@ -21,12 +21,12 @@ - ara-tox-py3 gate: jobs: + - ara-api-mysql - ansible-role-ara-api-fedora-devel - ansible-role-ara-api-fedora-2.9 - ansible-role-ara-api-ubuntu-2.8 - ansible-role-ara-api-ubuntu-2.7 - ansible-role-ara-api-ubuntu-postgresql - - ansible-role-ara-api-ubuntu-mysql - ansible-role-ara-api-fedora-distributed-sqlite - ansible-role-ara-web-ubuntu - ansible-role-ara-web-fedora diff --git a/.zuul.d/role-jobs.yaml b/.zuul.d/role-jobs.yaml index 83b1c28c..5ba7b679 100644 --- a/.zuul.d/role-jobs.yaml +++ b/.zuul.d/role-jobs.yaml @@ -83,14 +83,6 @@ pre-run: tests/install_docker.yaml run: tests/with_postgresql.yaml -- job: - name: ansible-role-ara-api-ubuntu-mysql - parent: ansible-role-ara-api-ubuntu - roles: - - zuul: zuul/zuul-jobs - pre-run: tests/install_docker.yaml - run: tests/with_mysql.yaml - - job: name: ansible-role-ara-api-fedora-distributed-sqlite parent: ansible-role-ara-api-fedora diff --git a/tests/vars/mysql_tests.yaml b/tests/vars/mysql_tests.yaml new file mode 100644 index 00000000..39752df8 --- /dev/null +++ b/tests/vars/mysql_tests.yaml @@ -0,0 +1,16 @@ +ara_tests_cleanup: true +ara_api_root_dir: "{{ ansible_user_dir }}/.ara-tests" +ara_api_secret_key: testing +ara_api_debug: true +ara_api_log_level: DEBUG +# Set to 0 because tests could be using the offline client +ara_api_database_conn_max_age: 0 +ara_api_database_engine: django.db.backends.mysql +ara_api_database_name: ara +ara_api_database_user: ara +ara_api_database_password: password +ara_api_database_port: 3306 +# The host is defined dynamically based on the address of the database server +# ara_api_database_host: 127.0.0.1 +_mysql_container_name: ara_tests_mariadb +_mysql_image_name: docker.io/mariadb:10 diff --git a/tests/with_mysql.yaml b/tests/with_mysql.yaml index e06662f4..e9536521 100644 --- a/tests/with_mysql.yaml +++ b/tests/with_mysql.yaml @@ -15,45 +15,21 @@ # You should have received a copy of the GNU General Public License # along with ARA. If not, see <http://www.gnu.org/licenses/>. -- name: Test the ARA API with mysql - hosts: all +- name: Start a mysql container + hosts: ara-database-server gather_facts: yes - vars: - _mysql_container_name: "ara_tests_mariadb" - _mysql_image_name: "mariadb:10" - ara_tests_cleanup: true - ara_api_root_dir: "{{ ansible_user_dir }}/.ara-tests" - ara_api_secret_key: testing - ara_api_debug: true - ara_api_log_level: DEBUG - ara_api_database_engine: django.db.backends.mysql - ara_api_database_name: ara - ara_api_database_user: ara - ara_api_database_password: password - ara_api_database_host: 127.0.0.1 - ara_api_database_port: 3306 - pre_tasks: - # TODO: This fails if the docker python library isn't installed but we can - # recover by running the command manually. The Ansible provided by Zuul - # does not have the module installed. - - name: Start a mariadb container - docker_container: - name: "{{ _mysql_container_name }}" - image: "{{ _mysql_image_name }}" - state: started - ports: - - "{{ ara_api_database_port }}:{{ ara_api_database_port }}" - env: - MYSQL_RANDOM_ROOT_PASSWORD: yes - MYSQL_DATABASE: "{{ ara_api_database_name }}" - MYSQL_USER: "{{ ara_api_database_user }}" - MYSQL_PASSWORD: "{{ ara_api_database_password }}" - ignore_errors: yes - register: _docker_run + vars_files: + - vars/mysql_tests.yaml + tasks: + - name: Install podman + become: yes + package: + name: podman + state: present - - name: Start a mysql container without docker python library + - name: Start a mysql container command: | - docker run -d \ + podman run -d \ --name {{ _mysql_container_name }} \ -p {{ ara_api_database_port }}:{{ ara_api_database_port }} \ -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ @@ -61,43 +37,55 @@ -e MYSQL_USER={{ ara_api_database_user }} \ -e MYSQL_PASSWORD={{ ara_api_database_password }} \ {{ _mysql_image_name }} - when: - - _docker_run is failed - - "'Failed to import docker or docker-py' in _docker_run.msg or 'Failed to import the required Python library' in _docker_run.msg" + + # podman doesn't appear to be able to listen on ipv6 yet: https://github.com/containers/libpod/issues/3245 + # If we have a node on IPv6, redirect the traffic from v6 to v4 with socat + - when: ansible_host | ipv6 != false + become: yes + block: + - name: Install socat + package: + name: socat + state: present + + - name: Setup systemd service + copy: + dest: /etc/systemd/system/socat-mysql.service + owner: root + group: root + mode: 0644 + content: | + [Unit] + Description=socat mysql ipv6 to ipv4 + + [Service] + ExecStart=/usr/bin/socat TCP6-LISTEN:3306,fork,bind=[{{ ansible_host }}] TCP4:127.0.0.1:3306 + + [Install] + WantedBy=multi-user.target + + - name: Start socat network redirection for mysql over ipv6 + service: + name: socat-mysql + state: started + enabled: yes + daemon_reload: yes + +- name: Deploy and test ARA API with mysql + hosts: ara-api-server + gather_facts: yes + vars_files: + - vars/mysql_tests.yaml tasks: - - block: - - name: Set up the API with the ara_api role - include_role: - name: ara_api - public: yes + - name: Set database server host + set_fact: + ara_api_database_host: "{{ hostvars['database-server']['ansible_host'] }}" - # These are tasks rather than a standalone playbook to give us an easy - # access to all the variables within the same play. - - include_tasks: test_tasks.yaml + - name: Set up the API with the ara_api Ansible role + include_role: + name: ara_api + public: yes - # Dump is suffixed by .txt so we don't need magic mimetypes when - # viewing on the web. - - name: Dump database file - shell: | - mysqldump \ - --host={{ ara_api_database_host }} \ - --port={{ ara_api_database_port }} \ - --user={{ ara_api_database_user }} \ - --password={{ ara_api_database_password }} \ - {{ ara_api_database_name }} > {{ ara_api_base_dir }}/mysqldump.sql.txt - always: - - when: ara_tests_cleanup | bool - block: - - name: Clean up the mysql container - docker_container: - name: "{{ _mysql_container_name }}" - state: absent - ignore_errors: yes - register: _docker_rm - - - name: Remove the mysql container without the docker python library - command: "docker rm -f {{ _mysql_container_name }}" - ignore_errors: yes - when: - - _docker_rm is failed - - "'Failed to import docker or docker-py' in _docker_rm.msg" + # These are tasks rather than a standalone playbook to give us an easy + # access to all the variables within the same play. + - include_tasks: test_tasks.yaml diff --git a/tests/zuul_post_logs.yaml b/tests/zuul_post_logs.yaml new file mode 100644 index 00000000..e8b61131 --- /dev/null +++ b/tests/zuul_post_logs.yaml @@ -0,0 +1,66 @@ +--- +# Copyright (c) 2020 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA Records Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA Records Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA Records Ansible. If not, see <http://www.gnu.org/licenses/>. + +- name: Recover API server data + hosts: ara-api-server + gather_facts: yes + vars: + ara_api_root_dir: "{{ ansible_user_dir }}/.ara-tests" + tasks: + # The zuul-output directory is retrieved by the fetch-output role + # https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/fetch-output + - name: Create log directory + file: + path: "{{ ansible_user_dir }}/zuul-output/logs" + state: directory + recurse: yes + + - name: Recover integration test data + command: cp -rp {{ ara_api_root_dir }}/server {{ ansible_user_dir }}/zuul-output/logs/server + ignore_errors: yes + + # Static report is generated with ara-manage generate in test_tasks.yaml + - name: Recover static report + command: cp -rp {{ ara_api_root_dir }}/static {{ ansible_user_dir }}/zuul-output/logs/static + ignore_errors: yes + + - name: Link the static report to the Zuul build with multiple nodes + zuul_return: + data: + zuul: + artifacts: + - name: "Static report for {{ inventory_hostname }}" + url: "{{ inventory_hostname }}/static/" + when: groups['all'] | length > 1 + + - name: Link the static report to the Zuul build with a single node + zuul_return: + data: + zuul: + artifacts: + - name: Static report + url: "static/" + when: groups['all'] | length == 1 + +- name: Generate Zuul ARA report + hosts: localhost + vars: + ara_report_type: html + ara_report_path: ara-report + roles: + - ara-report diff --git a/tests/zuul_post_with_mysql.yaml b/tests/zuul_post_with_mysql.yaml new file mode 100644 index 00000000..2e7486b1 --- /dev/null +++ b/tests/zuul_post_with_mysql.yaml @@ -0,0 +1,48 @@ +--- +# Copyright (c) 2020 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA Records Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA Records Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA Records Ansible. If not, see <http://www.gnu.org/licenses/>. + +- name: Recover mysql server data + hosts: ara-database-server + gather_facts: yes + vars_files: + - vars/mysql_tests.yaml + tasks: + - name: Ensure mysqldump is installed + become: yes + package: + name: mariadb + state: present + + # The zuul-output directory is retrieved by the fetch-output role + # https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/fetch-output + - name: Create log directory + file: + path: "{{ ansible_user_dir }}/zuul-output/logs" + state: directory + recurse: yes + + # Dump is suffixed by .txt so we don't need magic mimetypes when + # viewing on the web. + - name: Dump database file + shell: | + mysqldump \ + --host=127.0.0.1 \ + --port={{ ara_api_database_port }} \ + --user={{ ara_api_database_user }} \ + --password={{ ara_api_database_password }} \ + {{ ara_api_database_name }} > {{ ansible_user_dir }}/zuul-output/logs/mysqldump.sql.txt diff --git a/tests/zuul_pre_multinode_networking.yaml b/tests/zuul_pre_multinode_networking.yaml new file mode 100644 index 00000000..92f7f7e0 --- /dev/null +++ b/tests/zuul_pre_multinode_networking.yaml @@ -0,0 +1,25 @@ +--- +# Copyright (c) 2019 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA Records Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA Records Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA Records Ansible. If not, see <http://www.gnu.org/licenses/>. + +- name: Allow networking between multiple nodes + hosts: all + tasks: + # https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/multi-node-firewall + - name: Set up multi-node firewall + include_role: + name: multi-node-firewall