From e2686f9a5f2d006042b52f4b9b290661b5b9af90 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Thu, 27 Feb 2020 15:35:31 -0500 Subject: [PATCH] Add ovs flows monitoring This commit 1. Adds a custom plugin to get number of ovs flows on br-int 2. Adds a row/panel to the OpenStack General System Performance dashboard to visualize this 3. Adds support for containerized/non-containerized collectd 4. In case of containerized, ovs is installed on container and /etc/openvswitch and /var/run/openvswitch from host are mounted into container 5. The custom collectd exec plugin cannot be run as root, hence stack user is created within container and the plugin is run as stack user. In the case of non-containerized collectd, heat-admin user existing on overcloud nodes is used 6. Controllers and Computes are monitored Change-Id: I4687bd6aa0168c0b4bab72cf8eaca5f92c42b635 --- ansible/install/group_vars/all.yml | 1 + .../collectd-openstack/files/ovs_flows.sh | 9 ++++++++ .../roles/collectd-openstack/tasks/main.yml | 13 +++++++++++ .../templates/compute.collectd.conf.j2 | 7 ++++++ .../templates/controller.collectd.conf.j2 | 6 +++++ ansible/install/roles/collectd/tasks/main.yml | 5 ++++ .../templates/compute.collectd.conf.j2 | 7 ++++++ .../templates/controller.collectd.conf.j2 | 7 ++++++ ...enstack_general_system_performance.yaml.j2 | 5 ++++ .../templates/partials/ovs_flows.yaml | 23 +++++++++++++++++++ .../collectd-openstack/Dockerfile | 12 +++++++++- .../collectd-openstack/files/ovs_flows.sh | 8 +++++++ 12 files changed, 102 insertions(+), 1 deletion(-) create mode 100755 ansible/install/roles/collectd-openstack/files/ovs_flows.sh create mode 100644 ansible/install/roles/grafana-dashboards/templates/partials/ovs_flows.yaml create mode 100755 browbeat-containers/collectd-openstack/files/ovs_flows.sh diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 9d70febf6..593c40128 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -275,6 +275,7 @@ controller_monitored_queues: ovsagent_compute_monitor: false ovsagent_controller_monitor: false ovsagent_networker_monitor: false +ovs_flows_monitoring: false controller_monitored_ints: - "tap" diff --git a/ansible/install/roles/collectd-openstack/files/ovs_flows.sh b/ansible/install/roles/collectd-openstack/files/ovs_flows.sh new file mode 100755 index 000000000..bba5e1b67 --- /dev/null +++ b/ansible/install/roles/collectd-openstack/files/ovs_flows.sh @@ -0,0 +1,9 @@ +#!/bin/sh +HOSTNAME="${COLLECTD_HOSTNAME:-`hostname -f`}" +INTERVAL="${COLLECTD_INTERVAL:-15}" + +while sleep "$INTERVAL"; do + VALUE=$(sudo ovs-ofctl dump-flows br-int | wc -l) + echo "PUTVAL \"$HOSTNAME/ovs-flows/gauge-ovs_flows\" interval=$INTERVAL N:$VALUE" +done + diff --git a/ansible/install/roles/collectd-openstack/tasks/main.yml b/ansible/install/roles/collectd-openstack/tasks/main.yml index 633414e85..c2c795dee 100644 --- a/ansible/install/roles/collectd-openstack/tasks/main.yml +++ b/ansible/install/roles/collectd-openstack/tasks/main.yml @@ -245,6 +245,19 @@ dest: /usr/local/bin/collectd_ovsagent.py when: "('controller' in group_names ) or ('compute' in group_names) or ('networker' in group_names)" +- name: Copy exec plugins + copy: + src: "{{item.src}}" + dest: "{{item.dest}}" + owner: root + group: root + mode: 0755 + become: true + with_items: + - src: ovs_flows.sh + dest: /usr/local/bin/ovs_flows.sh + when: "('controller' in group_names ) or ('compute' in group_names)" + # Rabbitmq monitoring - name: Install pyrabbit easy_install: diff --git a/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 index d383f2d19..01af203c0 100644 --- a/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 @@ -123,6 +123,13 @@ PreCacheChain "PreCache" {% endif %} + +{% if ovs_flows_monitoring %} + + Exec "heat-admin:heat-admin" "/usr/local/bin/ovs_flows.sh" + +{% endif %} + {%if iostat_compute_collectd_plugin %} ModulePath "/usr/local/bin/" diff --git a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 index 75f6a74ae..9717a5d63 100644 --- a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 @@ -106,6 +106,12 @@ PreCacheChain "PreCache" {% if ceph_controller_collectd_plugin %} {% if inventory_hostname == groups['controller'][0] %} +{% if ovs_flows_monitoring %} + + Exec "heat-admin:heat-admin" "/usr/local/bin/ovs_flows.sh" + +{% endif %} + LongRunAvgLatency false ConvertSpecialMetricTypes true diff --git a/ansible/install/roles/collectd/tasks/main.yml b/ansible/install/roles/collectd/tasks/main.yml index 425b5846a..2ed24f943 100644 --- a/ansible/install/roles/collectd/tasks/main.yml +++ b/ansible/install/roles/collectd/tasks/main.yml @@ -87,5 +87,10 @@ {% if config_type == 'controller' or config_type == 'undercloud' %} -v /var/lib/mysql/mysql.sock:/var/lib/mysql/mysql.sock \ {% endif %} + {% if config_type == 'controller' or config_type == 'compute' %} + {% if ovs_flows_monitoring %} + -v /etc/openvswitch/:/etc/openvswitch/ -v /var/run/openvswitch/:/var/run/openvswitch/ \ + {% endif %} + {% endif %} collectd-openstack become: yes diff --git a/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 b/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 index f4f9ce958..ffd3f4a99 100644 --- a/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 +++ b/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 @@ -113,6 +113,13 @@ PreCacheChain "PreCache" {% endif %} + +{% if ovs_flows_monitoring %} + + Exec stack "/usr/local/bin/ovs_flows.sh" + +{% endif %} + {%if iostat_compute_collectd_plugin %} ModulePath "/usr/local/bin/" diff --git a/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 b/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 index adb6ec050..d9ab3df0d 100644 --- a/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 +++ b/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 @@ -200,6 +200,13 @@ PreCacheChain "PreCache" {% endif %} + +{% if ovs_flows_monitoring %} + + Exec stack "/usr/local/bin/ovs_flows.sh" + +{% endif %} + {%if iostat_controller_collectd_plugin %} ModulePath "/usr/local/bin/" diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 index 1e6e87df7..e76569d42 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 @@ -5,6 +5,7 @@ {% set ironic_groups = ['undercloud'] %} {% set mariadb_groups = ['undercloud', 'controller', '*'] %} {% set odl_groups = ['controller', '*'] %} +{% set ovsflows_groups = ['controller', 'compute', '*'] %} {% set ovsagent_groups = ['controller', 'compute', '*'] %} {% set rabbitmq_groups = ['undercloud', 'controller', '*'] %} {% set swift_stat_groups = ['controller', '*'] %} @@ -150,6 +151,10 @@ dashboard: {% include 'partials/ironic_metrics.yaml' %} {% endif %} +{% if item.template_node_type in ovsflows_groups %} + {% include 'partials/ovs_flows.yaml' %} +{% endif %} + {% if item.template_node_type in ovsagent_groups %} {% include 'partials/neutron_resources.yaml' %} {% endif %} diff --git a/ansible/install/roles/grafana-dashboards/templates/partials/ovs_flows.yaml b/ansible/install/roles/grafana-dashboards/templates/partials/ovs_flows.yaml new file mode 100644 index 000000000..90a8a7f64 --- /dev/null +++ b/ansible/install/roles/grafana-dashboards/templates/partials/ovs_flows.yaml @@ -0,0 +1,23 @@ + - title: OVS Flows + collapse: true + height: 200px + showTitle: true + panels: + - title: OVS Flows + type: graph + legend: + alignAsTable: true + avg: true + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: alias($Cloud.$Node.ovs-flows.gauge-ovs_flows, 'br-int flows') + yaxes: + - format: short + - format: short diff --git a/browbeat-containers/collectd-openstack/Dockerfile b/browbeat-containers/collectd-openstack/Dockerfile index 06a8ca94e..90b86a711 100644 --- a/browbeat-containers/collectd-openstack/Dockerfile +++ b/browbeat-containers/collectd-openstack/Dockerfile @@ -2,6 +2,7 @@ FROM centos:7 RUN yum update -y && \ yum clean all && \ + yum groupinstall -y development tools && \ yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ yum install -y centos-release-opstools && \ yum install -y collectd collectd-turbostat collectd-disk collectd-apache collectd-ceph \ @@ -9,13 +10,22 @@ RUN yum update -y && \ yum install -y sysstat && \ yum install -y python2-pip python2-devel && \ pip install pyrabbit && \ - yum install -y libdbi-dbd-mysql collectd-dbi + yum install -y libdbi-dbd-mysql collectd-dbi && \ + yum install -y https://cbs.centos.org/kojifiles/packages/openvswitch/2.11.0/4.el7/x86_64/openvswitch-2.11.0-4.el7.x86_64.rpm && \ + yum install -y libibverbs && \ + yum install -y sudo + +RUN useradd stack +RUN echo stack | passwd stack --stdin +RUN echo "stack ALL=(root) NOPASSWD:ALL" | tee -a /etc/sudoers.d/stack +RUN chmod 0440 /etc/sudoers.d/stack ADD files/collectd_ceph_storage.py /usr/local/bin/collectd_ceph_storage.py ADD files/collectd_gnocchi_status.py /usr/local/bin/collectd_gnocchi_status.py ADD files/collectd_rabbitmq_monitoring.py /usr/local/bin/collectd_rabbitmq_monitoring.py ADD files/collectd_swift_stat.py /usr/local/bin/collectd_swift_stat.py +ADD files/ovs_flows.sh /usr/local/bin/ovs_flows.sh ADD config/collectd.conf /etc/collectd.conf diff --git a/browbeat-containers/collectd-openstack/files/ovs_flows.sh b/browbeat-containers/collectd-openstack/files/ovs_flows.sh new file mode 100755 index 000000000..30e0e53b5 --- /dev/null +++ b/browbeat-containers/collectd-openstack/files/ovs_flows.sh @@ -0,0 +1,8 @@ +#!/bin/sh +HOSTNAME="${COLLECTD_HOSTNAME:-`hostname -f`}" +INTERVAL="${COLLECTD_INTERVAL:-15}" + +while sleep "$INTERVAL"; do + VALUE=$(sudo ovs-ofctl dump-flows br-int | wc -l) + echo "PUTVAL \"$HOSTNAME/ovs-flows/gauge-ovs_flows\" interval=$INTERVAL N:$VALUE" +done