x/browbeat
Code Issues Proposed changes

CFME/Satellite6 collectd/dashboard additions/fixes

Browse Source 
+ Fixed some grafana Postgresql dashboarding
+ Added background processes using cpu on Satellite6 to collectd and dashboards
+ Automated getting foreman/candlepin postgres passwords into collectd config file for satellite6

Change-Id: I17653500749b08ff56f416932384909f85c036a1
This commit is contained in:
Alex Krzos 2016-03-16 09:46:44 -04:00
parent 6bdc671b2f
commit dcb4d28a01
6 changed files with 697 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ansible/install/filter_plugins/browbeat_install_filters.py
View File

@ -21,6 +21,12 @@ def hosts_to_dictionary(arg):
return dictionary
def ini_value(key_value):
"""Strips key= from key=value from ini configuration data"""
equals_idx = key_value.index('=') + 1
return key_value[equals_idx:]
def to_grafana_refid(number):
"""Convert a number to a string starting at character a and incrementing. This only accounts
for a to zz, anything greater than zz is probably too much to graph anyway."""
@ -39,6 +45,7 @@ class FilterModule(object):
def filters(self):
return {
'dict_remove': dict_remove,
'ini_value': ini_value,
'hosts_to_dictionary': hosts_to_dictionary,
'to_grafana_refid': to_grafana_refid,
}

10
ansible/install/roles/collectd-generic/tasks/main.yml
View File

@ -18,6 +18,16 @@
become: true
with_items: "{{collectd_packages[config_type]}}"
- name: Get Candlepin PostgresDB Password
command: grep "jpa.config.hibernate.connection.password" /etc/candlepin/candlepin.conf
register: satellite_candlepin_password
when: "'{{config_type}}' == 'satellite6'"
- name: Get Foreman PostgresDB Password
shell: grep "password" /etc/foreman/database.yml | awk '{print $2}' | tr -d '"'
register: satellite_foreman_password
when: "'{{config_type}}' == 'satellite6'"
- name: Configure collectd.conf
template:
src={{config_type}}.collectd.conf.j2

64
ansible/install/roles/collectd-generic/templates/satellite6.collectd.conf.j2
View File

@ -21,7 +21,7 @@ LoadPlugin load
LoadPlugin memory
LoadPlugin numa
LoadPlugin processes
# LoadPlugin postgresql
LoadPlugin postgresql
LoadPlugin swap
LoadPlugin turbostat
LoadPlugin unixsock
@ -58,18 +58,52 @@ LoadPlugin uptime
IgnoreSelected false
</Plugin>
# <Plugin postgresql>
# <Database foreman>
# Host "localhost"
# Port "5432"
# User "root"
# </Database>
# </Plugin>
<Plugin postgresql>
<Database candlepin>
Host "localhost"
Port "5432"
User "candlepin"
Password "{{satellite_candlepin_password.stdout|ini_value}}"
# Predefined:
Query backends
Query transactions
Query queries
Query table_states
Query disk_io
Query disk_usage
</Database>
<Database foreman>
Host "localhost"
Port "5432"
User "foreman"
Password "{{satellite_foreman_password.stdout}}"
# Predefined:
Query backends
Query transactions
Query queries
Query table_states
Query disk_io
Query disk_usage
</Database>
<Database gutterball>
Host "localhost"
Port "5432"
User "gutterball"
Password "gutterball"
# Predefined:
Query backends
Query transactions
Query queries
Query table_states
Query disk_io
Query disk_usage
</Database>
</Plugin>
<Plugin processes>
# DynFlow
Process "dynflow_executor"
Process "dynflow_executor_monitor"
ProcessMatch "dynflow_executor" "dynflow.+executor$"
ProcessMatch "dynflow_executor_monitor" "dynflow.+executor.+monitor$"
# Elasticsearch
ProcessMatch "Elasticsearch" ".*elasticsearch.*"
@ -83,9 +117,9 @@ LoadPlugin uptime
# Passenger
ProcessMatch "Passenger-Forman" "Passenger.+RackApp:.+/usr/share/foreman"
ProcessMatch "Passenger-Puppet" "Passenger.+RackApp:.+/etc/puppet/rack"
Process "PassengerHelperAgent"
Process "PassengerLoggingAgent"
Process "PassengerWatchdog"
ProcessMatch "PassengerHelperAgent" "PassengerHelperAgent"
ProcessMatch "PassengerLoggingAgent" "PassengerLoggingAgent"
ProcessMatch "PassengerWatchdog" "PassengerWatchdog"
# Postgres database
ProcessMatch "postgres" "postgres.*"
@ -108,6 +142,10 @@ LoadPlugin uptime
# Tomcat
ProcessMatch "Tomcat" ".*tomcat.*"
# Other background tasks
ProcessMatch "Trends:Counter" "/opt/rh/ruby193/root/usr/bin/ruby.+/opt/rh/ruby193/root/usr/bin/rake.+trends:counter"
ProcessMatch "LDAP:Refresh_usergroups" "/opt/rh/ruby193/root/usr/bin/ruby.+/opt/rh/ruby193/root/usr/bin/rake.+ldap:refresh_usergroups"
Process "ruby"
# Collect on collectd process

29
ansible/install/roles/dashboard-generic/templates/cfme_general_system_performance.json.j2
View File

@ -1869,7 +1869,7 @@
"collapse": true,
"panels": [
{
"title": "$Cloud - $Node - vmdb_production Size",
"title": "$Cloud - $Node - Database Sizes",
"error": false,
"span": 12,
"editable": true,
@ -1927,7 +1927,7 @@
"targets": [
{
"refId": "A",
"target": "alias($Cloud.$Node.postgresql-vmdb_production.pg_db_size, 'Size')"
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_db_size, 'postgresql-', ''), 2)"
}
],
"aliasColors": {},
@ -1993,7 +1993,7 @@
"targets": [
{
"refId": "A",
"target": "alias($Cloud.$Node.postgresql-vmdb_production.pg_numbackends, 'Connected Clients')"
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_numbackends, 'postgresql-', ''), 2)"
}
],
"aliasColors": {},
@ -2001,7 +2001,7 @@
"links": []
},
{
"title": "$Cloud - $Node - Block Access",
"title": "$Cloud - $Node - Heap/Index/Toast Activity",
"error": false,
"span": 12,
"editable": true,
@ -2059,7 +2059,7 @@
"targets": [
{
"refId": "A",
"target": "aliasByNode($Cloud.$Node.postgresql-vmdb_production.pg_blks-*, 3)"
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-vmdb_production.pg_blks-*, 'pg_blks-', ''), 3)"
}
],
"aliasColors": {},
@ -2067,7 +2067,7 @@
"links": []
},
{
"title": "$Cloud - $Node - Block Access",
"title": "$Cloud - $Node - Tuple Operations",
"error": false,
"span": 12,
"editable": true,
@ -2125,7 +2125,7 @@
"targets": [
{
"refId": "A",
"target": "aliasByNode($Cloud.$Node.postgresql-vmdb_production.pg_n_tup_*, 3)"
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-vmdb_production.pg_n_tup_*, 'pg_n_tup_', ''), 3)"
}
],
"aliasColors": {},
@ -2191,7 +2191,7 @@
"targets": [
{
"refId": "A",
"target": "aliasByNode($Cloud.$Node.postgresql-vmdb_production.pg_scan-*, 3)"
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-vmdb_production.pg_scan-*, 'pg_scan-', ''), 3)"
}
],
"aliasColors": {},
@ -2199,13 +2199,14 @@
"links": []
},
{
"title": "$Cloud - $Node - Commits",
"title": "$Cloud - $Node - Commits/Rollbacks",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
"id": 381,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
@ -2256,7 +2257,11 @@
"targets": [
{
"refId": "A",
"target": "aliasByNode($Cloud.$Node.postgresql-vmdb_production.pg_xact-*, 3)"
"target": "aliasSub(aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_xact-commit, 'postgresql-', ''), 2), '$', '-commit')"
},
{
"refId": "B",
"target": "aliasSub(aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_xact-rollback, 'postgresql-', ''), 2), '$', '-rollback')"
}
],
"aliasColors": {},
@ -2332,7 +2337,7 @@
],
"showTitle": true
},
{
{
"collapse": true,
"editable": true,
"height": "250px",

607
ansible/install/roles/dashboard-generic/templates/satellite6_general_system_performance.json.j2
View File

@ -2290,6 +2290,613 @@
},
{% endfor %}
{# End Loop over per-process options here #}
{
"title": "Postgresql",
"height": "250px",
"editable": true,
"collapse": true,
"panels": [
{
"title": "$Cloud - $Node - Database Sizes",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"bits",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 1,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_db_size, 'postgresql-', ''), 2)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Connected Clients",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_numbackends, 'postgresql-', ''), 2)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Foreman Heap/Index/Toast Activity",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-foreman.pg_blks-*, 'pg_blks-', ''), 3)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Candlepin Heap/Index/Toast Activity",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-candlepin.pg_blks-*, 'pg_blks-', ''), 3)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Gutterball Heap/Index/Toast Activity",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-gutterball.pg_blks-*, 'pg_blks-', ''), 3)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Foreman Tuple Operations",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-foreman.pg_n_tup_*, 'pg_n_tup_', ''), 3)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Candlepin Tuple Operations",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-candlepin.pg_n_tup_*, 'pg_n_tup_', ''), 3)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Gutterball Tuple Operations",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasByNode(aliasSub($Cloud.$Node.postgresql-gutterball.pg_n_tup_*, 'pg_n_tup_', ''), 3)"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
},
{
"title": "$Cloud - $Node - Commits/Rollbacks",
"error": false,
"span": 12,
"editable": true,
"type": "graph",
"isNew": true,
{% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %}
"id": {{vars.panel_idx}},
"datasource": null,
"renderer": "flot",
"x-axis": true,
"y-axis": true,
"y_formats": [
"short",
"short"
],
"grid": {
"leftLogBase": 1,
"leftMax": null,
"rightMax": null,
"leftMin": null,
"rightMin": null,
"rightLogBase": 1,
"threshold1": null,
"threshold2": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"lines": true,
"fill": 0,
"linewidth": 2,
"points": false,
"pointradius": 5,
"bars": false,
"stack": false,
"percentage": false,
"legend": {
"show": true,
"values": true,
"min": true,
"max": true,
"current": true,
"total": false,
"avg": false,
"rightSide": true,
"alignAsTable": true
},
"nullPointMode": "connected",
"steppedLine": false,
"tooltip": {
"value_type": "cumulative",
"shared": true
},
"timeFrom": null,
"timeShift": null,
"targets": [
{
"refId": "A",
"target": "aliasSub(aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_xact-commit, 'postgresql-', ''), 2), '$', '-commit')"
},
{
"refId": "B",
"target": "aliasSub(aliasByNode(aliasSub($Cloud.$Node.postgresql-*.pg_xact-rollback, 'postgresql-', ''), 2), '$', '-rollback')"
}
],
"aliasColors": {},
"seriesOverrides": [],
"links": []
}
],
"showTitle": true
},
{
"collapse": true,
"editable": true,

6
ansible/install/roles/dashboard-generic/vars/main.yml
View File

@ -755,7 +755,7 @@ per_process_panels:
- Foreman-SmartProxy
- name: "Mongodb"
processes:
- Mongodb
- mongod
- name: "Passenger"
processes:
- Passenger-Forman
@ -783,6 +783,10 @@ per_process_panels:
- name: "Tomcat"
processes:
- Tomcat
- name: "Background Tasks"
processes:
- "Trends:Counter"
- "LDAP:Refresh_usergroups"
- name: "Collectd"
processes:
- collectd