diff --git a/browbeat-config.yaml b/browbeat-config.yaml index fb58767bc..e20a3ed70 100644 --- a/browbeat-config.yaml +++ b/browbeat-config.yaml @@ -57,6 +57,7 @@ rally: - browbeat: rally/rally-plugins/browbeat - workloads: rally/rally-plugins/workloads - dynamic-workloads: rally/rally-plugins/dynamic-workloads + - reports: rally/rally-plugins/reports shaker: server: 1.1.1.1 port: 5555 diff --git a/browbeat/workloads/rally.py b/browbeat/workloads/rally.py index b641d7f86..fdae99bb5 100644 --- a/browbeat/workloads/rally.py +++ b/browbeat/workloads/rally.py @@ -55,12 +55,12 @@ class Rally(base.WorkloadBase): for plugin in self.config['rally']['plugins']: for name in plugin: plugins.append(plugin[name]) - plugin_string = "" + self.plugin_string = "" if len(plugins) > 0: - plugin_string = "--plugin-paths {}".format(",".join(plugins)) + self.plugin_string = "--plugin-paths {}".format(",".join(plugins)) cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format( - plugin_string, task_file, task_args, test_name) + self.plugin_string, task_file, task_args, test_name) from_time = time.time() self.tools.run_cmd(cmd)['stdout'] to_time = time.time() @@ -79,18 +79,19 @@ class Rally(base.WorkloadBase): def gen_scenario_html(self, task_ids, test_name): all_task_ids = ' '.join(task_ids) cmd = "source {}; ".format(get_workload_venv('rally', True)) - cmd += "rally task report --uuid {} --out {}.html".format( - all_task_ids, test_name) + cmd += "rally {} task report --uuid {} --out {}.html".format( + self.plugin_string, all_task_ids, test_name) return self.tools.run_cmd(cmd)['stdout'] def gen_scenario_json(self, task_id): cmd = "source {}; ".format(get_workload_venv('rally', True)) - cmd += "rally task results --uuid {}".format(task_id) + cmd += "rally {} task results --uuid {}".format(self.plugin_string, task_id) return self.tools.run_cmd(cmd)['stdout'] def gen_scenario_json_file(self, task_id, test_name): cmd = "source {}; ".format(get_workload_venv('rally', True)) - cmd += "rally task results --uuid {} > {}.json".format(task_id, test_name) + cmd += "rally {} task results --uuid {} > {}.json".format(self.plugin_string, + task_id, test_name) return self.tools.run_cmd(cmd)['stdout'] def rally_metadata(self, result, meta): diff --git a/doc/source/images/Duplicate_Atomic_Actions_Duration_Line_Chart.png b/doc/source/images/Duplicate_Atomic_Actions_Duration_Line_Chart.png new file mode 100644 index 000000000..af8921fd0 Binary files /dev/null and b/doc/source/images/Duplicate_Atomic_Actions_Duration_Line_Chart.png differ diff --git a/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration1.png b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration1.png new file mode 100644 index 000000000..f055c271b Binary files /dev/null and b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration1.png differ diff --git a/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration2.png b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration2.png new file mode 100644 index 000000000..d1857b3e1 Binary files /dev/null and b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration2.png differ diff --git a/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration3.png b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration3.png new file mode 100644 index 000000000..7732f9bc8 Binary files /dev/null and b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration3.png differ diff --git a/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration4.png b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration4.png new file mode 100644 index 000000000..01f69fabd Binary files /dev/null and b/doc/source/images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration4.png differ diff --git a/doc/source/images/Resource_Atomic_Actions_Duration_Line_Chart.png b/doc/source/images/Resource_Atomic_Actions_Duration_Line_Chart.png new file mode 100644 index 000000000..95903202b Binary files /dev/null and b/doc/source/images/Resource_Atomic_Actions_Duration_Line_Chart.png differ diff --git a/doc/source/plugins.rst b/doc/source/plugins.rst index 3a76718db..dff86332c 100644 --- a/doc/source/plugins.rst +++ b/doc/source/plugins.rst @@ -44,3 +44,77 @@ Scenario - nova_boot_persist_with_network_volume_fip ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This scenario creates instances with a nic, a volume and associates a floating ip that persist upon completion of a rally run. It is used as a workload with Telemetry by spawning many instances that have many metrics for the Telemetry subsystem to collect upon. + +Charts +^^^^^^ + +To include any of the custom charts from Browbeat in a scenario, the following lines will have to be included in the python file of the program. +.. code-block:: python + + import sys + import os + + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../reports'))) + from generate_scenario_duration_charts import ScenarioDurationChartsGenerator # noqa: E402 + +The customc charts will appear in the "Scenario Data" section of the Rally HTML report. + +Chart - add_per_iteration_complete_data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This plugin generates a stacked area graph for duration trend for each atomic action in an iteration. +To include this chart in any scenario, add the following lines at the end of the run() function of the scenario in the python file. + +.. code-block:: python + + self.duration_charts_generator = ScenarioDurationChartsGenerator() + self.duration_charts_generator.add_per_iteration_complete_data(self) + +The graphs will appear under the "Per iteration" section of "Scenario Data" in the Rally HTML report. +The resulting graphs will look like the images below. + +.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration1.png + :alt: Iteration 1 Chart + +.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration2.png + :alt: Iteration 2 Chart + +.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration3.png + :alt: Iteration 3 Chart + +.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration4.png + :alt: Iteration 4 Chart + +Chart - add_duplicate_atomic_actions_iteration_additive_data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This plugin generates line graphs for atomic actions that have been executed more than once in the same iteration. +To include this chart in any scenario, add the following lines at the end of the run() function of the scenario in the python file. + +.. code-block:: python + + self.duration_charts_generator = ScenarioDurationChartsGenerator() + self.duration_charts_generator.add_duplicate_atomic_actions_iteration_additive_data(self) + +The graphs will appear under the "Aggregated" section of "Scenario Data" in the Rally HTML report. +The resulting graphs will look like the images below. + +.. image:: images/Duplicate_Atomic_Actions_Duration_Line_Chart.png + :alt: Duplicate Atomic Actions Duration Line Chart + +Chart - add_all_resources_additive_data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This plugin generates a line graph for duration data from each resource created by Rally. +To include this chart in any scenario, add the following lines at the end of the run() function of the scenario in the python file. + +.. code-block:: python + + self.duration_charts_generator = ScenarioDurationChartsGenerator() + self.duration_charts_generator.add_all_resources_additive_data(self) + +The graphs will appear under the "Aggregated" section of "Scenario Data" in the Rally HTML report. +The resulting graphs will look like the images below. + +.. image:: images/Resource_Atomic_Actions_Duration_Line_Chart.png + :alt: Resource Atomic Actions Duration Line Chart diff --git a/rally/rally-plugins/netcreate-boot/netcreate_nova_boot_vms_on_single_network.py b/rally/rally-plugins/netcreate-boot/netcreate_nova_boot_vms_on_single_network.py index 7c88689be..ef7f05376 100644 --- a/rally/rally-plugins/netcreate-boot/netcreate_nova_boot_vms_on_single_network.py +++ b/rally/rally-plugins/netcreate-boot/netcreate_nova_boot_vms_on_single_network.py @@ -17,6 +17,11 @@ from rally.task import scenario from rally.task import types from rally.task import validation +import sys +import os + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../reports'))) +from generate_scenario_duration_charts import ScenarioDurationChartsGenerator # noqa: E402 @types.convert(image={"type": "glance_image"}, flavor={"type": "nova_flavor"}) @validation.add("image_valid_on_flavor", flavor_param="flavor", image_param="image") @@ -38,3 +43,8 @@ class CreateVMsOnSingleNetwork(neutron_utils.NeutronScenario, port = self._create_port(network, port_create_args or {}) kwargs["nics"].append({'port-id': port['port']['id']}) self._boot_server(image, flavor, **kwargs) + + self.duration_charts_generator = ScenarioDurationChartsGenerator() + self.duration_charts_generator.add_per_iteration_complete_data(self) + self.duration_charts_generator.add_duplicate_atomic_actions_iteration_additive_data(self) + self.duration_charts_generator.add_all_resources_additive_data(self) diff --git a/rally/rally-plugins/reports/custom_chart_plugins.py b/rally/rally-plugins/reports/custom_chart_plugins.py new file mode 100644 index 000000000..ef38143e5 --- /dev/null +++ b/rally/rally-plugins/reports/custom_chart_plugins.py @@ -0,0 +1,81 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rally.common.plugin import plugin +from rally.task.processing import charts +from rally.task.processing import utils + +@plugin.configure(name="ResourceDurationLines") +class ResourceDurationOutputLinesChart(charts.OutputStackedAreaChart): + """Display resource duration data as generic chart with lines. + + This plugin processes additive data and displays it in HTML report + as linear chart with X axis bound to iteration number. + Complete output data is displayed as linear chart as well, without + any processing. + + Examples of using this plugin in Scenario, for saving output data: + + .. code-block:: python + + self.add_output( + additive={"title": "Resources atomic action duration line chart", + "description": "Resources trend", + "chart_plugin": "ResourceDurationLines", + "data": [["foo", 12], ["bar", 34]], + "label": "Duration(in seconds)", + "axis_label": "Resource count"}) + """ + + widget = "Lines" + + def add_iteration(self, iteration): + """Add iteration data. + This method must be called for each iteration. + :param iteration: list, resource duration data for current iteration + """ + atomic_count = {} + self.max_count = 0 + for name, value in iteration: + if name not in atomic_count.keys(): + atomic_count[name] = 1 + else: + atomic_count[name] += 1 + self.max_count = max(self.max_count, atomic_count[name]) + + for name, value in iteration: + if name not in self._data: + self._data[name] = utils.GraphZipper(self.base_size*self.max_count, + self.zipped_size) + self._data[name].add_point(value) + + def zeropad_duration_data(self): + """Some actions might have been executed more times than other actions + in the same iteration. Zeroes are appended to make the numbers equal. + """ + for key in self._data: + i = len(self._data[key].get_zipped_graph()) + while i < self.base_size*self.max_count: + i += 1 + self._data[key].add_point(0) + + def render(self): + """Render HTML from resource duration data""" + self.zeropad_duration_data() + render_data = [(name, points.get_zipped_graph()) + for name, points in self._data.items()] + return {"title": self.title, + "description": self.description, + "widget": self.widget, + "data": render_data, + "label": self.label, + "axis_label": self.axis_label} diff --git a/rally/rally-plugins/reports/generate_scenario_duration_charts.py b/rally/rally-plugins/reports/generate_scenario_duration_charts.py new file mode 100644 index 000000000..1ea73adb9 --- /dev/null +++ b/rally/rally-plugins/reports/generate_scenario_duration_charts.py @@ -0,0 +1,69 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from process_atomic_actions_data import AtomicActionsDurationDataProcessor + +class ScenarioDurationChartsGenerator(AtomicActionsDurationDataProcessor): + """Send processed complete and additive atomic action duration data to Rally""" + + def add_per_iteration_complete_data(self, scenario_object): + """Generate a stacked area graph for duration trend for each atomic action + in an iteration. + :param scenario_object: Rally scenario object + """ + atomic_actions = scenario_object.atomic_actions() + self.process_atomic_actions_complete_data(atomic_actions) + # Some actions might have been executed more times than other actions + # in the same iteration. Zeroes are appended to make the numbers equal. + self.zeropad_duration_data() + scenario_object.add_output(complete={ + "title": "Atomic actions duration data as stacked area", + "description": "Iterations trend", + "chart_plugin": "StackedArea", + "data": ( + self.get_complete_duration_data()), + "label": "Duration(in seconds)", + "axis_label": "Atomic action"}) + + def add_duplicate_atomic_actions_iteration_additive_data(self, scenario_object): + """Generate line graphs for atomic actions that have been executed more than once + in the same iteration. + :param scenario_object: Rally scenario object + """ + atomic_actions = scenario_object.atomic_actions() + for action_name in self.get_duplicate_actions_list(atomic_actions): + additive_data_duplicate_action = ( + self.process_atomic_action_additive_data(action_name, + atomic_actions)) + scenario_object.add_output(additive={ + "title": "{} additive duration data as line chart".format( + action_name), + "description": "Iterations trend", + "chart_plugin": "Lines", + "data": additive_data_duplicate_action, + "label": "Duration(in seconds)"}) + + def add_all_resources_additive_data(self, scenario_object): + """Generate a line graph for duration data from each resource created by Rally. + :param scenario_object: Rally scenario object + """ + atomic_actions = scenario_object.atomic_actions() + additive_data_all_actions = [] + for action in atomic_actions: + additive_data_all_actions.append([action["name"], action["finished_at"] - + action["started_at"]]) + scenario_object.add_output(additive={"title": "Resources atomic action duration line chart", + "description": "Resources trend", + "chart_plugin": "ResourceDurationLines", + "data": additive_data_all_actions, + "label": "Duration(in seconds)", + "axis_label": "Resource count"}) diff --git a/rally/rally-plugins/reports/process_atomic_actions_data.py b/rally/rally-plugins/reports/process_atomic_actions_data.py new file mode 100644 index 000000000..d609f2c0b --- /dev/null +++ b/rally/rally-plugins/reports/process_atomic_actions_data.py @@ -0,0 +1,77 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class AtomicActionsDurationDataProcessor: + """Generate charts for atomic actions durations additive and complete data""" + + def __init__(self): + self._duration_data = {} + self.max_num_data_points = 0 + + def process_atomic_actions_complete_data(self, atomic_actions): + """Generate duration data in complete format for per iteration chart + :param atomic_actions: list, self.atomic_actions() from a rally scenario + """ + for action in atomic_actions: + if action["name"] not in self._duration_data: + self._duration_data[action["name"]] = [] + action_duration = action["finished_at"] - action["started_at"] + self._duration_data[action["name"]].append([len(self._duration_data[action["name"]]), + action_duration]) + self.max_num_data_points = max(self.max_num_data_points, + len(self._duration_data[action["name"]])) + + def get_duplicate_actions_list(self, atomic_actions): + """Get list of atomic actions which occur more than once in an iteration + :param atomic_actions: list, self.atomic_actions() from a rally scenario + :returns: list of strings representing duplicate action names + """ + actions_set = set() + duplicate_actions_list = [] + for action in atomic_actions: + if action["name"] not in actions_set: + actions_set.add(action["name"]) + else: + duplicate_actions_list.append(action["name"]) + return duplicate_actions_list + + def process_atomic_action_additive_data(self, action_name, atomic_actions): + """Generate duration data in additive format for aggregate chart + :param action_name: str, action name to generate duration data for + :param atomic_actions: list, self.atomic_actions() from a rally scenario + :returns: list in Rally additive data format + """ + additive_duration_data = [] + action_index = 1 + for action in atomic_actions: + if action["name"] == action_name: + additive_duration_data.append(["{}({})".format(action_name, action_index), + action["finished_at"] - action["started_at"]]) + action_index += 1 + return additive_duration_data + + def zeropad_duration_data(self): + """Some atomic actions occur more times than other atomic actions + within the same iteration. Zeroes are appended to make the length + of all atomic action lists the same + """ + for action_name in self._duration_data: + while len(self._duration_data[action_name]) < self.max_num_data_points: + self._duration_data[action_name].append([len(self._duration_data[action_name]), 0]) + + def get_complete_duration_data(self): + """Complete duration data is stored in dict format to increase efficiency + of operations. Rally add_output() function expects a list as input. This + function converts the complete duration data to the format expected by the + Rally add_output() function. + """ + return [[name, durations] for name, durations in self._duration_data.items()]