From cf024be5350260a849c9262ecff9550c5a510ff0 Mon Sep 17 00:00:00 2001 From: lpiwowar Date: Mon, 30 Aug 2021 16:17:18 +0200 Subject: [PATCH] Fix test discovery by checktest.py Chectest.py wasn't able to catch bug [1] as it didn't take into consideration inheritance during the test discovery. The cause of the issue is the fact that chectest.py parsed python files by itself to discover the tests. This patch fixes this by using unittest library for test discovery instead. [1] https://review.opendev.org/c/osf/interop/+/806178 Story: 2009146 Task: 43097 Change-Id: I6e1b11eeb3ca1915ca41b6af88f9f568e6d674eb --- tools/checktests.py | 72 +++++++++++++++++++++++++++----------------- tools/consistency.sh | 27 +++++++++-------- 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/tools/checktests.py b/tools/checktests.py index eeac86a3..e321cde3 100644 --- a/tools/checktests.py +++ b/tools/checktests.py @@ -1,4 +1,5 @@ # Copyright 2018, OpenStack Foundation +# Copyright 2021, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -13,12 +14,12 @@ # under the License. import argparse -import ast import importlib import json import os import re import sys +import unittest def get_v1_version(guideline): @@ -90,7 +91,7 @@ def get_submodules(parent_module_name): if not os.path.exists(os.path.join(root, '__init__.py')): continue for f in files: - if f.endswith('.py'): + if f.endswith('.py') and not f == "__init__.py": module_path = root + '/' + f module_name = root_name + '.' + os.path.splitext(f)[0] if module_name not in submodules: @@ -98,26 +99,42 @@ def get_submodules(parent_module_name): return submodules -def get_tests(module_name): - submodules = get_submodules(module_name) - tests = {} - for module_name in submodules: - filename = submodules[module_name] - with open(filename, 'r') as f: - source = f.read() - parsed = ast.parse(source) - for node in parsed.body: - if node.__class__ is ast.ClassDef: - for classnode in node.body: - if (classnode.__class__ is ast.FunctionDef and - classnode.name.startswith('test_')): - for decorator in classnode.decorator_list: - if hasattr(decorator, 'func'): - if decorator.func.attr == 'idempotent_id': - tests['id-' + decorator.args[0].s] = \ - module_name + "." + node.name + "." + \ - classnode.name - return tests +def get_module_tests(tests, parsed_test): + if isinstance(tests, unittest.TestCase): + test_description = tests.id() + test_uuid_regex = r'id-\w{8}-\w{4}-\w{4}-\w{4}-\w{12}' + test_id = re.search(test_uuid_regex, test_description) + + if not test_id: + return parsed_test + + test_id = test_id.group(0) + test_name = test_description.split("[")[0] + test_list = parsed_test.get(test_id, []) + test_list.append(test_name) + parsed_test[test_id] = test_list + + return parsed_test + elif not isinstance(tests, unittest.suite.TestSuite): + return + + for test in tests: + parsed_test = get_module_tests(test, parsed_test) + + return parsed_test + + +def get_tests(submodules): + loader = unittest.TestLoader() + parsed_tests = {} + for submodule in submodules: + try: + tests = loader.loadTestsFromName(submodule) + parsed_tests = get_module_tests(tests, parsed_tests) + except Exception as e: + print("Unable to load: {}. Exception: {}".format(submodule, e)) + + return parsed_tests def run(): @@ -133,10 +150,10 @@ def run(): 'against') args = parser.parse_args() - guideline = load_guideline(args.guideline_file) required = get_required_tests(guideline) - tests = get_tests(args.testlib) + submodules = get_submodules(args.testlib) + lib_tests = get_tests(submodules) missing_uuids = [] missing_tests = {} @@ -144,10 +161,11 @@ def run(): for test in required: uuid = test[0] testnames = test[1] - if uuid not in tests: + if uuid not in lib_tests: missing_uuids.append(test) else: - if tests[uuid] not in testnames: + in_testnames = [test in lib_tests[uuid] for test in testnames] + if not any(in_testnames): missing_tests[uuid] = test exit_code = 0 @@ -182,7 +200,7 @@ def run(): " idempotent_id:\n" " %s\n" " names: " % (args.testlib, - uuid, tests[uuid], + uuid, lib_tests[uuid], args.guideline_file, missing_tests[uuid][0])) for testname in missing_tests[uuid][1]: diff --git a/tools/consistency.sh b/tools/consistency.sh index f69713f0..3a42e1bb 100755 --- a/tools/consistency.sh +++ b/tools/consistency.sh @@ -77,17 +77,21 @@ if [[ -z $SFSDIR ]]; then git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR fi +pip install $TEMPESTDIR +pip install $DNSDIR +pip install $ORCHESTRATIONDIR +pip install $SFSDIR export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR python3 ./tools/checktests.py --guideline guidelines/next.json exit_1=$? -python3 ./tools/checktests.py --guideline add-ons/guidelines/dns.next.json --testlib designate_tempest_plugin -exit_2=$? -# TODO(kopecmartin) In order to unblock gates, skip check of manila tempest plugin until the following bug is resolved: -# https://storyboard.openstack.org/#!/story/2009146 -# python3 ./tools/checktests.py --guideline add-ons/guidelines/shared_file_system.next.json --testlib manila_tempest_tests -# exit_3=$? +# TODO(lpiwowar) The consistency check of designate_tempest_plugin is omitted until +# https://bugs.launchpad.net/designate/+bug/1943115 is fixed. +# python3 ./tools/checktests.py --guideline add-ons/guidelines/dns.next.json --testlib designate_tempest_plugin +# exit_2=$? +python3 ./tools/checktests.py --guideline add-ons/guidelines/shared_file_system.next.json --testlib manila_tempest_tests +exit_3=$? # TODO(kopecmartin) consistency check of heat_tempest_plugin is omitted intentionally until we improve the # checktests.py so that it detects ids of the heat_tempest_plugin.api tests which don't use decorator.idempotent_id # call to track the id @@ -96,10 +100,10 @@ exit_2=$? python3 ./tools/checktests.py --guideline current_guideline exit_5=$? -python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin -exit_6=$? -# python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests -# exit_7=$? +# python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin +# exit_6=$? +python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests +exit_7=$? # python3 ./tools/checktests.py --guideline add-ons/orchestration_current_guideline --testlib heat_tempest_plugin # exit_8=$? @@ -111,6 +115,5 @@ if [[ "${CLEANTEMPEST}" ]]; then rm -rf $SFSDIR fi - #! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 )) -! (( $exit_1 || $exit_2 || $exit_5 || $exit_6 )) +! (( $exit_1 || $exit_3 || $exit_5 || $exit_7 ))