From 74defc22734439cb0be563a4fbc647f02a8f7575 Mon Sep 17 00:00:00 2001
From: David Moreau Simard <moi@dmsimard.com>
Date: Sun, 6 Sep 2020 11:25:42 -0400
Subject: [PATCH] API: Stop returning nested children resources
When querying the API for a playbook's detail, it would return all of
it's children (hosts, files, tasks, results) which could be very slow
when dealing with larger playbooks.
We no longer do that for playbooks as well as plays and tasks.
Instead, we can easily find a playbook's resources by searching for them
with the playbook id like so:
- /api/v1/plays?playbook=<id>
- /api/v1/tasks?playbook=<id>
- /api/v1/results?playbook=<id>
... and so on.
This commit adapts the built-in UI because it would've otherwise been
broken by the change.
Fixes: https://github.com/ansible-community/ara/issues/158
Change-Id: I442bff657e5da9d6a3916ebdbc5e66c0e670b00f
---
ara/api/serializers.py | 105 ++-------------------------------
ara/ui/templates/playbook.html | 16 ++---
ara/ui/views.py | 32 +++++++++-
doc/source/api-usage.rst | 41 +++++++------
tests/integration/lookups.yaml | 5 --
5 files changed, 64 insertions(+), 135 deletions(-)
diff --git a/ara/api/serializers.py b/ara/api/serializers.py
index f6656a01..bfbfd33e 100644
--- a/ara/api/serializers.py
+++ b/ara/api/serializers.py
@@ -74,8 +74,7 @@ class FileSha1Serializer(serializers.ModelSerializer):
#######
-# Simple serializers provide lightweight representations of objects without
-# nested or large fields.
+# Simple serializers provide lightweight representations of objects suitable for inclusion in other objects
#######
@@ -96,108 +95,27 @@ class SimplePlaybookSerializer(ItemCountSerializer):
class SimplePlaySerializer(ItemCountSerializer):
class Meta:
model = models.Play
- exclude = ("uuid", "created", "updated")
+ exclude = ("playbook", "uuid", "created", "updated")
class SimpleTaskSerializer(ItemCountSerializer, TaskPathSerializer):
class Meta:
model = models.Task
- exclude = ("tags", "created", "updated")
+ exclude = ("playbook", "play", "created", "updated")
-
-class SimpleResultSerializer(ResultStatusSerializer):
- class Meta:
- model = models.Result
- exclude = ("content", "created", "updated")
+ tags = ara_fields.CompressedObjectField(read_only=True)
class SimpleHostSerializer(serializers.ModelSerializer):
class Meta:
model = models.Host
- exclude = ("facts", "created", "updated")
+ exclude = ("playbook", "facts", "created", "updated")
class SimpleFileSerializer(FileSha1Serializer):
class Meta:
model = models.File
- exclude = ("content", "created", "updated")
-
-
-class SimpleRecordSerializer(serializers.ModelSerializer):
- class Meta:
- model = models.Record
- exclude = ("value", "created", "updated")
-
-
-#######
-# Nested serializers returns optimized data within the context of another object.
-# For example: when retrieving a playbook, we'll already have the playbook id
-# so it is not necessary to include it in nested objects.
-#######
-
-
-class NestedPlaybookFileSerializer(serializers.ModelSerializer):
- class Meta:
- model = models.File
- exclude = ("content", "created", "updated", "playbook")
-
-
-class NestedPlaybookHostSerializer(serializers.ModelSerializer):
- class Meta:
- model = models.Host
- fields = ("id", "name")
-
-
-class NestedPlaybookResultSerializer(ResultStatusSerializer):
- class Meta:
- model = models.Result
- exclude = ("content", "created", "updated", "playbook", "play", "task")
-
- host = NestedPlaybookHostSerializer(read_only=True)
-
-
-class NestedPlaybookTaskSerializer(serializers.ModelSerializer):
- class Meta:
- model = models.Task
- exclude = ("playbook", "created", "updated")
-
- tags = ara_fields.CompressedObjectField(read_only=True)
- file = NestedPlaybookFileSerializer(read_only=True)
- results = serializers.SerializerMethodField()
-
- @staticmethod
- def get_results(obj):
- results = obj.results.all().order_by("-id")
- return NestedPlaybookResultSerializer(results, many=True).data
-
-
-class NestedPlaybookRecordSerializer(serializers.ModelSerializer):
- class Meta:
- model = models.Record
- exclude = ("playbook", "value", "created", "updated")
-
-
-class NestedPlaybookPlaySerializer(serializers.ModelSerializer):
- class Meta:
- model = models.Play
- exclude = ("playbook", "uuid", "created", "updated")
-
- tasks = serializers.SerializerMethodField()
-
- @staticmethod
- def get_tasks(obj):
- tasks = obj.tasks.all().order_by("-id")
- return NestedPlaybookTaskSerializer(tasks, many=True).data
-
-
-class NestedPlayTaskSerializer(TaskPathSerializer):
- class Meta:
- model = models.Task
- exclude = ("playbook", "play", "created", "updated")
-
- tags = ara_fields.CompressedObjectField(read_only=True)
- results = NestedPlaybookResultSerializer(read_only=True, many=True)
- file = NestedPlaybookFileSerializer(read_only=True)
+ exclude = ("playbook", "content", "created", "updated")
#######
@@ -219,15 +137,6 @@ class DetailedPlaybookSerializer(ItemCountSerializer):
arguments = ara_fields.CompressedObjectField(default=ara_fields.EMPTY_DICT, read_only=True)
labels = SimpleLabelSerializer(many=True, read_only=True, default=[])
- hosts = SimpleHostSerializer(many=True, read_only=True, default=[])
- files = SimpleFileSerializer(many=True, read_only=True, default=[])
- records = NestedPlaybookRecordSerializer(many=True, read_only=True, default=[])
- plays = serializers.SerializerMethodField()
-
- @staticmethod
- def get_plays(obj):
- plays = obj.plays.all().order_by("-id")
- return NestedPlaybookPlaySerializer(plays, many=True).data
class DetailedPlaySerializer(ItemCountSerializer):
@@ -236,7 +145,6 @@ class DetailedPlaySerializer(ItemCountSerializer):
fields = "__all__"
playbook = SimplePlaybookSerializer(read_only=True)
- tasks = NestedPlayTaskSerializer(many=True, read_only=True, default=[])
class DetailedTaskSerializer(ItemCountSerializer, TaskPathSerializer):
@@ -247,7 +155,6 @@ class DetailedTaskSerializer(ItemCountSerializer, TaskPathSerializer):
playbook = SimplePlaybookSerializer(read_only=True)
play = SimplePlaySerializer(read_only=True)
file = SimpleFileSerializer(read_only=True)
- results = NestedPlaybookResultSerializer(many=True, read_only=True, default=[])
tags = ara_fields.CompressedObjectField(read_only=True)
diff --git a/ara/ui/templates/playbook.html b/ara/ui/templates/playbook.html
index a2d26f07..ac2b241c 100644
--- a/ara/ui/templates/playbook.html
+++ b/ara/ui/templates/playbook.html
@@ -7,7 +7,7 @@
<div class="pf-c-card__body">
<details id="records">
<summary>Records</summary>
- {% if playbook.items.records %}
+ {% if records %}
<ul class="pf-c-list">
{% for record in playbook.records %}
<li><a href="../records/{{ record.id }}.html">{{ record.key }}</a></li>
@@ -20,7 +20,7 @@
<details id="files">
<summary>Files</summary>
<ul class="pf-c-list">
- {% for file in playbook.files %}
+ {% for file in files %}
<li><a href="../files/{{ file.id }}.html">{{ file.path }}</a></li>
{% endfor %}
</ul>
@@ -74,7 +74,7 @@
</tr>
</thead>
<tbody>
- {% for host in playbook.hosts %}
+ {% for host in hosts %}
<tr>
<td role="cell" data-label="Hostname" class="pf-m-fit-content">
<a href="../hosts/{{ host.id }}.html">{{ host.name }}</a>
@@ -117,9 +117,7 @@
</tr>
</thead>
<tbody>
- {% for play in playbook.plays %}
- {% for task in play.tasks %}
- {% for result in task.results %}
+ {% for result in results %}
<tr role="row">
<td role="cell" data-label="Status" class="pf-c-table__icon pf-m-fit-content">
{% include "partials/result_status_icon.html" with status=result.status %}
@@ -128,10 +126,10 @@
{{ result.host.name }}
</td>
<td role="cell" data-label="Name" class="pf-m-fit-content">
- <a href="../results/{{ result.id }}.html">{{ task.name }}</a>
+ <a href="../results/{{ result.id }}.html">{{ result.task.name }}</a>
</td>
<td role="cell" data-label="Action" class="pf-m-fit-content">
- <a href="../files/{{ task.file.id }}.html#line-{{ task.lineno }}">{{ task.action }}</a>
+ <a href="../files/{{ task.file.id }}.html#line-{{ result.task.lineno }}">{{ result.task.action }}</a>
</td>
<td role="cell" data-label="Duration" class="pf-m-fit-content">
{{ result.duration | format_duration }}
@@ -141,8 +139,6 @@
</td>
</tr>
{% endfor %}
- {% endfor %}
- {% endfor %}
</tbody>
</table>
</details>
diff --git a/ara/ui/views.py b/ara/ui/views.py
index e4d374d3..00fe5623 100644
--- a/ara/ui/views.py
+++ b/ara/ui/views.py
@@ -68,9 +68,35 @@ class Playbook(generics.RetrieveAPIView):
template_name = "playbook.html"
def get(self, request, *args, **kwargs):
- playbook = self.get_object()
- serializer = serializers.DetailedPlaybookSerializer(playbook)
- return Response({"playbook": serializer.data})
+ playbook = serializers.DetailedPlaybookSerializer(self.get_object())
+ hosts = serializers.ListHostSerializer(
+ models.Host.objects.filter(playbook=playbook.data["id"]).all(), many=True
+ )
+ files = serializers.ListFileSerializer(
+ models.File.objects.filter(playbook=playbook.data["id"]).all(), many=True
+ )
+ records = serializers.ListRecordSerializer(
+ models.Record.objects.filter(playbook=playbook.data["id"]).all(), many=True
+ )
+ results = serializers.ListResultSerializer(
+ models.Result.objects.filter(playbook=playbook.data["id"]).all(), many=True
+ )
+
+ for result in results.data:
+ task_id = result["task"]
+ result["task"] = serializers.SimpleTaskSerializer(models.Task.objects.get(pk=task_id)).data
+ host_id = result["host"]
+ result["host"] = serializers.SimpleHostSerializer(models.Host.objects.get(pk=host_id)).data
+
+ # fmt: off
+ return Response({
+ "playbook": playbook.data,
+ "hosts": hosts.data,
+ "files": files.data,
+ "records": records.data,
+ "results": results.data
+ })
+ # fmt: on
class Host(generics.RetrieveAPIView):
diff --git a/doc/source/api-usage.rst b/doc/source/api-usage.rst
index a58d4216..7ea721de 100644
--- a/doc/source/api-usage.rst
+++ b/doc/source/api-usage.rst
@@ -82,25 +82,30 @@ Here's a code example to help you get started:
# If there are any results from our query, get more information about the
# failure and print something helpful
template = "{timestamp}: {host} failed '{task}' ({task_file}:{lineno})"
- for playbook in playbooks["results"]:
- # Get a detailed version of the playbook that provides additional context
- detailed_playbook = client.get("/api/v1/playbooks/%s" % playbook["id"])
- # Iterate through the playbook to get the context
- # Playbook -> Play -> Task -> Result <- Host
- for play in detailed_playbook["plays"]:
- for task in play["tasks"]:
- for result in task["results"]:
- if result["status"] in ["failed", "unreachable"]:
- print(template.format(
- timestamp=result["ended"],
- host=result["host"]["name"],
- task=task["name"],
- task_file=task["file"]["path"],
- lineno=task["lineno"]
- ))
+ for playbook in playbooks["results"]:
+ # Get failed results for the playbook
+ results = client.get("/api/v1/results?playbook=%s" % playbook["id"])
+
+ # For each result, print the task and host information
+ for result in results["results"]:
+ task = client.get("/api/v1/tasks/%s" % result["task"])
+ host = client.get("/api/v1/hosts/%s" % result["host"])
+
+ print(template.format(
+ timestamp=result["ended"],
+ host=host["name"],
+ task=task["name"],
+ task_file=task["path"],
+ lineno=task["lineno"]
+ ))
Running this script would then provide an output that looks like the following::
- 2019-03-20T16:18:41.710765: localhost failed 'smoke-tests : Return false' (tests/integration/roles/smoke-tests/tasks/test-ops.yaml:25)
- 2019-03-20T16:19:17.332663: localhost failed 'fail' (tests/integration/failed.yaml:22)
+ 2020-04-18T17:16:13.394056Z: aio1_repo_container-0c92f7a2 failed 'repo_server : Install EPEL gpg keys' (/home/zuul/src/opendev.org/openstack/openstack-ansible-repo_server/tasks/repo_install.yml:16)
+ 2020-04-18T17:14:59.930995Z: aio1_repo_container-0c92f7a2 failed 'repo_server : File and directory setup (root user)' (/home/zuul/src/opendev.org/openstack/openstack-ansible-repo_server/tasks/repo_pre_install.yml:78)
+ 2020-04-18T17:14:57.909155Z: aio1_repo_container-0c92f7a2 failed 'repo_server : Git service data folder setup' (/home/zuul/src/opendev.org/openstack/openstack-ansible-repo_server/tasks/repo_pre_install.yml:70)
+ 2020-04-18T17:14:57.342091Z: aio1_repo_container-0c92f7a2 failed 'repo_server : Check if the git folder exists already' (/home/zuul/src/opendev.org/openstack/openstack-ansible-repo_server/tasks/repo_pre_install.yml:65)
+ 2020-04-18T17:14:56.793499Z: aio1_repo_container-0c92f7a2 failed 'repo_server : Drop repo pre/post command script' (/home/zuul/src/opendev.org/openstack/openstack-ansible-repo_server/tasks/repo_pre_install.yml:53)
+ 2020-04-18T17:14:54.507660Z: aio1_repo_container-0c92f7a2 failed 'repo_server : File and directory setup (non-root user)' (/home/zuul/src/opendev.org/openstack/openstack-ansible-repo_server/tasks/repo_pre_install.yml:32)
+ 2020-04-18T17:14:51.281530Z: aio1_repo_container-0c92f7a2 failed 'repo_server : Create the nginx system user' (/home/zuul/src/opendev.org/openstack/openstack-ansible-repo_server/tasks/repo_pre_install.yml:22)
diff --git a/tests/integration/lookups.yaml b/tests/integration/lookups.yaml
index 0b4ca780..9ea2c8d9 100644
--- a/tests/integration/lookups.yaml
+++ b/tests/integration/lookups.yaml
@@ -26,11 +26,6 @@
- playbook.ansible_version == ansible_version.full
- playbook_dir in playbook.path
- "'tests/integration/lookups.yaml' in playbook.path"
- - "playbook.files | length == playbook['items']['files']"
- - "playbook.hosts | length == playbook['items']['hosts']"
- - "playbook.plays | length == playbook['items']['plays']"
- - "tasks.results | length == playbook['items']['tasks']"
- - "results.results | length == playbook['items']['results']"
#####
# Examples taken from docs on Ansible plugins and use cases