Merge "Add scripts to build and run ARA API containers"

.zuul.d/jobs.yaml

@@ -151,3 +151,11 @@
       - name: ara_api_credentials
         secret: ara_api_demo_credentials
         pass-to-parent: true
+
+- job:
+    name: ara-container-images
+    parent: ara-integration-base
+    nodeset: ara-fedora-31
+    description: |
+      Builds ARA API container images with buildah and briefly tests them with podman.
+    run: tests/with_container_images.yaml

.zuul.d/project.yaml

@@ -14,6 +14,7 @@
             voting: false
         - ara-basic-ansible-2.8
         - ara-basic-ansible-2.7
+        - ara-container-images
         - ara-tox-linters
         - ara-tox-py3
     gate:
@@ -23,6 +24,7 @@
         - ara-api-postgresql
         - ara-basic-ansible-2.8
         - ara-basic-ansible-2.7
+        - ara-container-images
         - ara-tox-linters
         - ara-tox-py3
     post:

contrib/container-images/README.rst

@@ -0,0 +1,200 @@
+Running ARA API server container images
+=======================================
+
+The ARA API server is a good candidate for being served out of a container as
+the configuration and state can be kept in persistent files and databases.
+
+The project maintains `different scripts <https://github.com/ansible-community/ara/tree/master/contrib/container-images>`_
+that are used to build and push simple container images to
+`DockerHub <https://hub.docker.com/repository/docker/recordsansible/ara-api>`_.
+
+The scripts are designed to yield images that are opinionated and
+"batteries-included" for the sake of simplicity.
+They install the necessary packages for connecting to MySQL and PostgreSQL
+databases and set up gunicorn as the application server.
+
+You are encouraged to use these scripts as a base example that you can build,
+tweak and improve the container image according to your specific needs and
+preferences.
+
+For example, precious megabytes can be saved by installing only the things you
+need and you can change the application server as well as it's configuration.
+
+Building an image with buildah
+------------------------------
+
+You will need to install `buildah <https://github.com/containers/buildah/blob/master/install.md>`_.
+
+The different scripts to build container images are available in the
+`git source repository <https://github.com/ansible-community/ara/tree/master/contrib/container-images>`_:
+
+- ``fedora-distribution.sh``: Builds an image from Fedora 32 `distribution packages <https://koji.fedoraproject.org/koji/packageinfo?packageID=24394>`_
+- ``fedora-pypi.sh``: Builds an image from `PyPi <https://pypi.org/project/ara>`_ packages on Fedora 32
+- ``fedora-source.sh``: Builds an image from `git source <https://github.com/ansible-community/ara>`_ on Fedora 32
+
+The scripts have no arguments other than the ability to specify an optional name
+and tag:
+
+.. code-block:: bash
+
+    $ git clone https://github.com/ansible-community/ara
+    $ cd ara/contrib/container-images
+    $ ./fedora-source.sh ara-api:latest
+    # [...]
+    Getting image source signatures
+    Copying blob 59bbb69efd73 skipped: already exists
+    Copying blob ccc3e7c17eae done
+    Copying config fb679fc301 done
+    Writing manifest to image destination
+    Storing signatures
+    fb679fc301dde7007b4d219f1d30060b3b4b0d5883b030ee7058d7e9f5969fbe
+
+The image will be available for use once the script has finished running:
+
+.. code-block:: bash
+
+    $ buildah images
+    REPOSITORY          TAG      IMAGE ID       CREATED          SIZE
+    localhost/ara-api   latest   fb679fc301dd   25 minutes ago   451 MB
+
+Running an image with podman
+----------------------------
+
+You will need to install `podman <https://podman.io/getting-started/installation>`_.
+
+Once an image has been built with the scripts above, you can validate that it
+is available to podman:
+
+.. code-block:: bash
+
+    $ podman images
+    REPOSITORY          TAG      IMAGE ID       CREATED          SIZE
+    localhost/ara-api   latest   fb679fc301dd   31 minutes ago   451 MB
+
+First, create a directory where settings, logs and sqlite databases will
+persist inside a volume and then start the container:
+
+.. code-block:: bash
+
+    $ mkdir -p ~/.ara/server
+    $ podman run --name api-server --detach --tty \
+         --volume ~/.ara/server:/opt/ara:z -p 8000:8000 \
+         localhost/ara-api
+    bc4b7630c265bdac161f2e08116f3f45c2db519fb757ddf865bb0f212780fa8d
+
+You can validate if the container is running properly with podman:
+
+.. code-block:: bash
+
+    $ podman ps
+    CONTAINER ID  IMAGE                     COMMAND               CREATED         STATUS             PORTS                   NAMES
+    bc4b7630c265  localhost/ara-api:latest  /usr/bin/gunicorn...  12 seconds ago  Up 11 seconds ago  0.0.0.0:8000->8000/tcp  api-server
+
+At this point, the API server is running but if it is your first time launching
+it, it will not be able to accept requests until you run initial database
+migrations for the default sqlite backend:
+
+.. code-block:: bash
+
+    $ podman exec -it api-server ara-manage migrate
+    [ara] Using settings file: /opt/ara/settings.yaml
+    Operations to perform:
+    Apply all migrations: admin, api, auth, contenttypes, db, sessions
+    Running migrations:
+    Applying contenttypes.0001_initial... OK
+    Applying auth.0001_initial... OK
+    Applying admin.0001_initial... OK
+    Applying admin.0002_logentry_remove_auto_add... OK
+    Applying admin.0003_logentry_add_action_flag_choices... OK
+    Applying api.0001_initial... OK
+    Applying api.0002_remove_host_alias... OK
+    Applying api.0003_add_missing_result_properties... OK
+    Applying api.0004_duration_in_database... OK
+    Applying api.0005_unique_label_names... OK
+    Applying contenttypes.0002_remove_content_type_name... OK
+    Applying auth.0002_alter_permission_name_max_length... OK
+    Applying auth.0003_alter_user_email_max_length... OK
+    Applying auth.0004_alter_user_username_opts... OK
+    Applying auth.0005_alter_user_last_login_null... OK
+    Applying auth.0006_require_contenttypes_0002... OK
+    Applying auth.0007_alter_validators_add_error_messages... OK
+    Applying auth.0008_alter_user_username_max_length... OK
+    Applying auth.0009_alter_user_last_name_max_length... OK
+    Applying auth.0010_alter_group_name_max_length... OK
+    Applying auth.0011_update_proxy_permissions... OK
+    Applying db.0001_initial... OK
+    Applying sessions.0001_initial... OK
+
+Once SQL migrations have been run, the API server should be reachable at
+http://127.0.0.1:8000 but it'll be empty.
+
+Data must be sent to it by running an Ansible playbook with the ARA callback
+installed and configured to use this API server.
+
+Sending data to the API server
+------------------------------
+
+Here's an example of how it works:
+
+.. code-block:: bash
+
+    # Create and source a python3 virtual environment
+    python3 -m venv ~/.ara/virtualenv
+    source ~/.ara/virtualenv/bin/activate
+
+    # Install Ansible and ARA
+    pip3 install ansible ara
+
+    # Configure Ansible to know where ARA's callback plugin is located
+    export ANSIBLE_CALLBACK_PLUGINS=$(python3 -m ara.setup.callback_plugins)
+
+    # Set up the ARA callback to know where the API server is
+    export ARA_API_CLIENT=http
+    export ARA_API_SERVER="http://127.0.0.1:8000"
+
+    # Run any of your Ansible playbooks as you normally would
+    ansible-playbook playbook.yml
+
+As each task from the playbook starts and completes, their data will be
+available on the API server in real time as you refresh your queries.
+
+Common operations
+-----------------
+
+Modifying ARA's API server settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Settings for the API server will be found in ``~/.ara/server/settings.yaml``
+(or ``/opt/ara/settings.yaml`` inside the container) and modifications are
+effective after a container restart:
+
+.. code-block:: bash
+
+    podman restart api-server
+
+See the `documentation <https://ara.readthedocs.io/en/latest/api-configuration.html>`_
+for the full list of available options.
+
+Running outside of localhost
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To run an API server that can be queried from other hosts, edit
+``~/.ara/server/settings.yaml`` and add the desired hostname (or IP) in
+`ALLOWED_HOSTS <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-allowed-hosts>`_.
+
+Connecting to mysql, mariadb or postgresql backends
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ARA API server is a good candidate for living in a container because the
+state can be stored on remote database servers.
+
+To connect to database backends other than the sqlite default, edit
+``~/.ara/server/settings.yaml`` and look for the following settings:
+
+- `DATABASE_ENGINE <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-database-engine>`_
+- `DATABASE_NAME <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-database-name>`_
+- `DATABASE_USER <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-database-user>`_
+- `DATABASE_PASSWORD <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-database-password>`_
+- `DATABASE_HOST <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-database-host>`_
+- `DATABASE_PORT <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-database-port>`_
+- `DATABASE_CONN_MAX_AGE <https://ara.readthedocs.io/en/latest/api-configuration.html#ara-database-conn-max-age>`_

contrib/container-images/fedora-distribution.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -x
+# Builds an ARA API server container image from Fedora 32 distribution packages.
+build=$(buildah from fedora:32)
+
+# Get all updates, install the ARA API server, database backends and gunicorn application server
+# This lets users swap easily from the sqlite default to mysql or postgresql just by tweaking settings.yaml.
+buildah run "${build}" -- /bin/bash -c "dnf update -y && dnf install -y ara ara-server python3-psycopg2 python3-mysql python3-gunicorn && dnf clean all"
+
+# Set up the container to run the API server with gunicorn and expose the port
+buildah config --env ARA_BASE_DIR=/opt/ara "${build}"
+buildah config --cmd "/usr/bin/gunicorn-3 --workers=4 --access-logfile - --bind 0.0.0.0:8000 ara.server.wsgi" "${build}"
+buildah config --port 8000 "${build}"
+
+# Commit this container to an image name
+buildah commit "${build}" "${1:-$USER/ara-api}"

contrib/container-images/fedora-pypi.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -x
+# Builds an ARA API server container image using the latest PyPi packages on Fedora 32.
+build=$(buildah from fedora:32)
+
+# Get all updates, install pip, database backends and gunicorn application server
+# This lets users swap easily from the sqlite default to mysql or postgresql just by tweaking settings.yaml.
+# Note: We use the packaged versions of psycopg2 and mysql python libraries so
+#       we don't need to install development libraries before installing them from PyPi.
+buildah run "${build}" -- /bin/bash -c "dnf update -y && dnf install -y python3-pip python3-psycopg2 python3-mysql python3-gunicorn && dnf clean all"
+
+# Install ara from source with API server extras for dependencies (django & django-rest-framework)
+buildah run "${build}" -- /bin/bash -c "pip3 install ara[server]"
+
+# Set up the container to run the API server with gunicorn and expose the port
+buildah config --env ARA_BASE_DIR=/opt/ara "${build}"
+buildah config --cmd "/usr/bin/gunicorn-3 --workers=4 --access-logfile - --bind 0.0.0.0:8000 ara.server.wsgi" "${build}"
+buildah config --port 8000 "${build}"
+
+# Commit this container to an image name
+buildah commit "${build}" "${1:-$USER/ara-api}"

contrib/container-images/fedora-source.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -x
+# Builds an ARA API server container image from source on Fedora 32.
+# TODO: Instead of cloning, it should probably use the source that's already checked out
+#       which would make it easier to test the script itself.
+SOURCE="https://github.com/ansible-community/ara"
+TMPDIR=$(mktemp -d)
+
+# Clone the source to a temporary directory and generate an sdist tarball we can install from
+git clone ${SOURCE} ${TMPDIR}/ara
+pushd ${TMPDIR}/ara
+python3 setup.py sdist
+sdist=$(ls dist/ara-*.tar.gz)
+popd
+
+build=$(buildah from fedora:32)
+
+# Get all updates, install pip, database backends and gunicorn application server
+# This lets users swap easily from the sqlite default to mysql or postgresql just by tweaking settings.yaml.
+# Note: We use the packaged versions of psycopg2 and mysql python libraries so
+#       we don't need to install development libraries before installing them from PyPi.
+buildah run "${build}" -- /bin/bash -c "dnf update -y && dnf install -y python3-pip python3-psycopg2 python3-mysql python3-gunicorn && dnf clean all"
+
+# Install ara from source with API server extras for dependencies (django & django-rest-framework)
+buildah run --volume ${TMPDIR}/ara:/usr/local/src/ara:z "${build}" -- /bin/bash -c "pip3 install /usr/local/src/ara/${sdist}[server]"
+
+# Set up the container to run the API server with gunicorn and expose the port
+buildah config --env ARA_BASE_DIR=/opt/ara "${build}"
+buildah config --cmd "/usr/bin/gunicorn-3 --workers=4 --access-logfile - --bind 0.0.0.0:8000 ara.server.wsgi" "${build}"
+buildah config --port 8000 "${build}"
+
+# Commit this container to an image name
+buildah commit "${build}" "${1:-$USER/ara-api}"

doc/source/container-images.rst

@@ -0,0 +1,3 @@
+.. _container-images:
+
+.. include:: ../../contrib/container-images/README.rst

doc/source/index.rst

@@ -21,6 +21,7 @@ Table of Contents
     Setting playbook names and labels <playbook-names-and-labels>
     Recording arbitrary data in playbooks <ara-record>
     Querying ARA from inside playbooks <ara-api-lookup>
+    Running ARA API server container images <container-images>
     Contributing to ARA <contributing>
 
 .. toctree::

tests/container_test_tasks.yaml

@@ -0,0 +1,89 @@
+- name: Ensure volume directory exists
+  file:
+    path: "{{ ara_api_root_dir }}/server"
+    state: directory
+    recurse: yes
+
+- name: "Build {{ item.name }}:{{ item.tag }} with {{ item.script }}"
+  command: "{{ ara_api_source }}/contrib/container-images/{{ item.script }} {{ item.name }}:{{ item.tag }}"
+
+- name: "Start {{ item.name }}:{{ item.tag }} with podman"
+  command: |
+    podman run --name api-server --detach --tty \
+    --volume {{ ara_api_root_dir }}/server:/opt/ara:z -p 8000:8000 \
+    {{ item.name }}:{{ item.tag }}
+
+- name: Run SQL migrations
+  command: podman exec -it api-server ara-manage migrate
+  register: _sql_migrations
+  # Allow the container to settle from booting up
+  retries: 3
+  delay: 5
+  until: _sql_migrations is succeeded
+
+- block:
+    - name: Get the API root
+      uri:
+        url: "http://127.0.0.1:8000/api/"
+        return_content: yes
+        follow_redirects: none
+        method: GET
+      register: _get_root
+      # Allow the server to settle from sql migrations
+      until: _get_root.status == 200
+      retries: 3
+      delay: 5
+
+    - name: Validate the API response
+      assert:
+        that:
+          - "'gunicorn' in _get_root.server"
+          - _get_root.json["kind"] == "ara"
+          - _get_root.json["api"] == ["http://127.0.0.1:8000/api/v1/"]
+
+    - name: Create a test playbook
+      uri:
+        url: "http://127.0.0.1:8000/api/v1/playbooks"
+        return_content: yes
+        follow_redirects: none
+        method: POST
+        status_code: 201
+        body_format: json
+        body:
+          name: "Integration test playbook for {{ item.script }}"
+          ansible_version: "9.0.0.1"
+          started: "{{ ansible_date_time.iso8601_micro }}"
+          status: running
+          labels:
+            - "{{ _get_root.json['version'] }}"
+            - "{{ item.name }}:{{ item.tag }}"
+            - "{{ item.script }}"
+          path: "/tests/container_test_tasks.yaml"
+      register: _post_playbook
+
+    - name: Get the test playbook
+      uri:
+        url: "http://127.0.0.1:8000/api/v1/playbooks/{{ _post_playbook.json['id'] }}"
+        return_content: yes
+        follow_redirects: none
+        method: GET
+        status_code: 200
+      register: _get_playbook
+
+    - name: Assert the test playbook
+      assert:
+        that:
+          - _get_playbook.json["id"] == _post_playbook.json["id"]
+  always:
+    - name: Delete previous static build
+      file:
+        path: "{{ ara_api_root_dir }}/server/static"
+        state: absent
+
+    - name: Generate a static report
+      command: podman exec -it api-server ara-manage generate /opt/ara/static
+      ignore_errors: yes
+
+    # The container gets removed but the data persists in ~/.ara-tests/server
+    - name: Stop the container and remove it
+      command: podman rm -f api-server

tests/with_container_images.yaml

@@ -0,0 +1,64 @@
+#  Copyright (c) 2020 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+- name: Test container images
+  hosts: all
+  gather_facts: yes
+  vars:
+    ara_api_root_dir: "{{ ansible_user_dir }}/.ara-tests"
+    ara_api_source: "{{ ansible_user_dir }}/src/opendev.org/recordsansible/ara"
+    images:
+      # These are in chronological order of release so that we don't end up
+      # running SQL migrations backwards during the tests.
+      - name: localhost/ara-api
+        tag: distribution-latest
+        script: fedora-distribution.sh
+      - name: localhost/ara-api
+        tag: pypi-latest
+        script: fedora-pypi.sh
+      - name: localhost/ara-api
+        tag: source-latest
+        script: fedora-source.sh
+  tasks:
+    - name: Install git, buildah and podman
+      become: yes
+      package:
+        name:
+          - git
+          - buildah
+          - podman
+        state: present
+
+    # TODO: Troubleshoot permission denied issues when running
+    #       ara-manage generate from container
+    - when: ansible_os_family == "RedHat"
+      block:
+        - name: Install python3-libselinux
+          become: yes
+          package:
+            name: python3-libselinux
+            state: present
+
+        - name: Set selinux to permissive
+          become: yes
+          selinux:
+            policy: targeted
+            state: permissive
+
+    - name: Test each container image
+      include_tasks: container_test_tasks.yaml
+      loop: "{{ images }}"