# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import PerfKit
import Rally
import Shaker
import Yoda
import logging
import os
import subprocess
import yaml
import re
from pykwalify import core as pykwalify_core
from pykwalify import errors as pykwalify_errors
class Tools(object):
def __init__(self, config=None):
self.logger = logging.getLogger('browbeat.Tools')
self.config = config
return None
# Returns true if ping successful, false otherwise
def is_pingable(self, ip):
cmd = "ping -c1 " + ip
result = self.run_cmd(cmd)
if result['rc'] == 0:
return True
else:
return False
# Run command async from the python main thread, return Popen handle
def run_async_cmd(self, cmd):
FNULL = open(os.devnull, 'w')
self.logger.debug("Running command : %s" % cmd)
process = subprocess.Popen(cmd, shell=True, stdout=FNULL)
return process
# Run command, return stdout as result
def run_cmd(self, cmd):
self.logger.debug("Running command : %s" % cmd)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
output_dict = {}
output_dict['stdout'] = stdout.strip()
output_dict['stderr'] = stderr.strip()
output_dict['rc'] = process.returncode
if process.returncode > 0:
self.logger.error("Command {} returned with error".format(cmd))
self.logger.error("stdout: {}".format(stdout))
self.logger.error("stderr: {}".format(stderr))
return output_dict
# Find Command on host
def find_cmd(self, cmd):
_cmd = "which %s" % cmd
self.logger.debug('Find Command : Command : %s' % _cmd)
command = self.run_cmd(_cmd)['stdout']
if command is None:
self.logger.error("Unable to find %s" % cmd)
raise Exception("Unable to find command : '%s'" % cmd)
return False
else:
return command.strip()
# Create directory for results
def create_results_dir(self, *args):
the_directory = '/'.join(args)
if not os.path.isdir(the_directory):
try:
os.makedirs(the_directory)
except OSError as err:
self.logger.error(
"Error creating the results directory: {}".format(err))
return False
return the_directory
def _load_config(self, path, validate=True):
try:
stream = open(path, 'r')
except IOError:
self.logger.error(
"Configuration file {} passed is missing".format(path))
exit(1)
config = yaml.safe_load(stream)
stream.close()
self.config = config
if validate:
self.validate_yaml()
return config
def validate_yaml(self):
self.logger.info(
"Validating the configuration file passed by the user")
stream = open("lib/validate.yaml", 'r')
schema = yaml.safe_load(stream)
check = pykwalify_core.Core(
source_data=self.config, schema_data=schema)
try:
check.validate(raise_exception=True)
self.logger.info("Validation successful")
except pykwalify_errors.SchemaError as e:
self.logger.error("Schema Validation failed")
raise Exception('File does not conform to schema: {}'.format(e))
def _run_workload_provider(self, provider):
self.logger = logging.getLogger('browbeat')
if provider == "perfkit":
perfkit = PerfKit.PerfKit(self.config)
perfkit.start_workloads()
elif provider == "rally":
rally = Rally.Rally(self.config)
rally.start_workloads()
elif provider == "shaker":
shaker = Shaker.Shaker(self.config)
shaker.run_shaker()
elif provider == "yoda":
yoda = Yoda.Yoda(self.config)
yoda.start_workloads()
else:
self.logger.error("Unknown workload provider: {}".format(provider))
def check_metadata(self):
meta = self.config['elasticsearch']['metadata_files']
for _meta in meta:
if not os.path.isfile(_meta['file']):
self.logger.error(
"Metadata file {} is not present".format(_meta['file']))
return False
return True
def gather_metadata(self):
os.putenv("ANSIBLE_SSH_ARGS",
" -F {}".format(self.config['ansible']['ssh_config']))
ansible_cmd = \
'ansible-playbook -i {} {}' \
.format(self.config['ansible']['hosts'], self.config['ansible']['metadata'])
self.run_cmd(ansible_cmd)
if not self.check_metadata():
self.logger.warning("Metadata could not be gathered")
return False
else:
self.logger.info("Metadata about cloud has been gathered")
return True
def post_process(self, cli):
workloads = {}
workloads['shaker'] = re.compile("shaker")
workloads['perfkit'] = re.compile("perfkit")
workloads['rally'] = re.compile("(?!perfkit)|(?!shaker)")
""" Iterate through dir structure """
results = {}
if os.path.isdir(cli.path):
for dirname, dirnames, files in os.walk(cli.path):
self.logger.info("Inspecting : %s" % dirname)
results[dirname] = files
else:
self.logger.error("Path does not exist")
return False
""" Capture per-workload results """
workload_results = {}
json = re.compile("\.json")
if len(results) > 0:
for path in results:
for regex in workloads:
if re.findall(workloads[regex], path):
if regex not in workload_results:
workload_results[regex] = []
for file in results[path]:
if (re.findall(json, file) and
'result_index-es' not in file):
workload_results[regex].append(
"{}/{}".format(path, file))
else:
self.logger.error("Results are empty")
return False
""" Iterate through each workload result, generate ES JSON """
if len(workload_results) > 0:
for workload in workload_results:
if workload is "rally":
rally = Rally.Rally(self.config)
for file in workload_results[workload]:
errors, results = rally.file_to_json(file)
if workload is "shaker":
# Stub for Shaker.
continue
if workload is "perfkit":
# Stub for PerfKit.
continue
def load_stackrc(self, filepath):
values = {}
with open(filepath) as stackrc:
for line in stackrc:
pair = line.split('=')
if 'export' not in line and '#' not in line and '$(' not in line:
values[pair[0].strip()] = pair[1].strip()
elif '$(' in line and 'for key' not in line:
values[pair[0].strip()] = \
self.run_cmd("echo " + pair[1].strip())['stdout'].strip()
return values