
This update packages the report tool and plugin files into Debian, and bundles it with the collect tool so that they are added to the 'collect' tarballs at the time of creation. The report tool now allows users to point it at any collect bundle and have it automatically extract the tarball and tar files for each host before running. This update also adds heartbeat loss, maintenance errors, daemon failures, and state changes plugin algorithms to the report tool. Some of the existing algorithms were enhanced to extract more relevant log events. The alarm algorithm was updated to only track when alarms switch between set and clear. Lastly, there is a correlator function implemented into the tool that determines failures in collect bundles and their root causes, as well as finds significant events and state changes from the log files. The number of times alarms are set and cleared are also counted and printed by the correlator. They are presented in output files and summaries are printed out onto the command line. Users can also specify if they want the correlator to only find events, alarm transitions and state changes for a specific host. Test Plan: PASS: Verify tool is packaged in Debian PASS: Verify tool is inserted into 'collect' tarballs PASS: Verify tool extracts tarballs and host tarfiles PASS: Verify tool can point at any collect bundle and run successfully PASS: Verify substring plugin algorithm is working PASS: Verify swact activity plugin algorithm is working PASS: Verify heartbeat loss plugin algorithm is working PASS: Verify maintenance errors plugin algorithm is working PASS: Verify daemon failures plugin algorithm is working PASS: Verify state changes plugin algorithm is working PASS: Verify alarm plugin algorithm is working PASS: Verify failures and correct root causes are found by correlator PASS: Verify significant events are found by correlator PASS: Verify alarm transitions are found by correlator PASS: Verify state changes are found by correlator PASS: Verify failures/events/alarms/state changes are printed into files PASS: Verify tool prints correct info onto command line PASS: Verify correlator only finds events for specified host PASS: Verify correlator only finds alarm transitions for specified host PASS: Verify correlator only finds state changes for specified host Story: 2010166 Task: 46177 Signed-off-by: Angela Mao <Angela.Mao@windriver.com> Change-Id: I02e28edf16b342abf2224cc98325d77ba0678055
195 lines
6.7 KiB
Python
Executable File
195 lines
6.7 KiB
Python
Executable File
########################################################################
|
|
#
|
|
# Copyright (c) 2022 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
########################################################################
|
|
#
|
|
# This file contains the Plugin class.
|
|
# The Plugin class contains all the labels and information of a plugin.
|
|
#
|
|
# Plugins contain labels to instruct the execution engine what to search
|
|
# for and where to search.
|
|
#
|
|
########################################################################
|
|
|
|
|
|
from datetime import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
|
|
import algorithms
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Plugin:
|
|
def __init__(self, file="", opts=None):
|
|
"""Constructor for the Plugin class
|
|
|
|
Parameters:
|
|
file (string) : Absolute filepath of the plugin
|
|
opts (dictionary): Options from command line when running algorithm
|
|
"""
|
|
self.file = file
|
|
self.opts = opts
|
|
self.state = {
|
|
"algorithm": None,
|
|
"files": [],
|
|
"hosts": [],
|
|
"substring": [],
|
|
"alarm_exclude": [],
|
|
"entity_exclude": [],
|
|
"start": None,
|
|
"end": None,
|
|
}
|
|
if file:
|
|
try:
|
|
self._file_set_attributes()
|
|
except KeyError as e:
|
|
raise e
|
|
elif opts:
|
|
self._opts_set_attributes()
|
|
|
|
try:
|
|
self.verify()
|
|
except ValueError as e:
|
|
raise e
|
|
|
|
def _file_set_attributes(self):
|
|
"""Sets plugin attributes from plugin files"""
|
|
with open(self.file) as f:
|
|
for line in f:
|
|
try:
|
|
self.extract(line)
|
|
except Exception as e:
|
|
raise e
|
|
|
|
def _opts_set_attributes(self):
|
|
"""Sets plugin attributes from command line options"""
|
|
for k, v in self.opts.items():
|
|
self.state[k] = v
|
|
|
|
def extract(self, line):
|
|
"""Extracts and sets attributes for this plugin
|
|
|
|
Parameters:
|
|
line (string): Line from plugin file to extract
|
|
"""
|
|
|
|
# split string from first '=', left side is label right side is value
|
|
data = line.strip().split("=", 1)
|
|
if len(data) <= 1:
|
|
raise ValueError("Value not specified for label")
|
|
label = data[0]
|
|
value = data[1]
|
|
label = label.replace(" ", "")
|
|
try:
|
|
if label == "algorithm":
|
|
self.state["algorithm"] = value.replace(" ", "")
|
|
elif label == "substring":
|
|
self.state["substring"].append(data[1])
|
|
elif label == "hosts":
|
|
self.state["hosts"] = value.replace(" ", "").split(",")
|
|
elif label == "alarm_exclude":
|
|
self.state["alarm_exclude"] = value.replace(" ", "").split(",")
|
|
elif label == "entity_exclude":
|
|
self.state["entity_exclude"] = value.replace(
|
|
" ", "").split(",")
|
|
elif label == "files":
|
|
self.state["files"] = value.replace(" ", "").split(",")
|
|
elif label == "start":
|
|
self.state["start"] = value
|
|
elif label == "end":
|
|
self.state["end"] = value
|
|
else:
|
|
logger.warning("unknown label: %s", label)
|
|
|
|
except KeyError:
|
|
logger.warning("unknown label: %s", label)
|
|
|
|
def verify(self):
|
|
"""Verify if this plugin's attributes are viable
|
|
|
|
Errors:
|
|
ValueError if a value is incorrectly set
|
|
"""
|
|
|
|
plugin_name = os.path.basename(self.file)
|
|
HOSTS_ERR = f"plugin: {plugin_name} should not have hosts specified"
|
|
|
|
if self.state["algorithm"] == algorithms.SUBSTRING:
|
|
self.validate_state(plugin_name, "files")
|
|
self.validate_state(plugin_name, "hosts")
|
|
self.validate_state(plugin_name, "substring")
|
|
elif self.state["algorithm"] == algorithms.ALARM:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.SYSTEM_INFO:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.SWACT_ACTIVITY:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.PUPPET_ERRORS:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.PROCESS_FAILURES:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.HEARTBEAT_LOSS:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.MAINTENANCE_ERR:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.DAEMON_FAILURES:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.STATE_CHANGES:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
elif self.state["algorithm"] == algorithms.AUDIT:
|
|
if len(self.state["hosts"]) > 0:
|
|
raise ValueError(HOSTS_ERR)
|
|
|
|
try:
|
|
datetime.strptime(self.state["start"], "%Y-%m-%d %H:%M:%S")
|
|
except:
|
|
raise ValueError(
|
|
f"plugin : {plugin_name} needs a start time in YYYY-MM-DD "
|
|
f"HH:MM:SS format"
|
|
)
|
|
|
|
try:
|
|
datetime.strptime(self.state["end"], "%Y-%m-%d %H:%M:%S")
|
|
except:
|
|
raise ValueError(
|
|
f"plugin : {plugin_name} needs an end time in YYYY-MM-DD "
|
|
f"HH:MM:SS format"
|
|
)
|
|
else:
|
|
raise ValueError(
|
|
f"plugin: {plugin_name} unknown algorithm "
|
|
f"{self.state['algorithm']}"
|
|
)
|
|
|
|
for host in self.state["hosts"]:
|
|
if host not in ["controllers", "workers", "storages", "all"]:
|
|
raise ValueError(
|
|
f"host not recognized: '{host}', accepted hosts are "
|
|
f"'controllers', 'workers', 'storages', 'all'"
|
|
)
|
|
|
|
def validate_state(self, plugin_name, key):
|
|
if len(self.state[key]) == 0:
|
|
raise ValueError(
|
|
f"plugin: {plugin_name} needs {key} specified for "
|
|
f"substring algorithm"
|
|
)
|
|
|
|
def __str__(self) -> str:
|
|
return f"{json.dumps(self.state)} File: {self.file}"
|