openstack/kolla-ansible
Code Issues Proposed changes

Remove "distutils" library

Browse Source 
Library "distutils" is deprecated in Python 3.10:
https://peps.python.org/pep-0632/

The versions previously referenced using StrictVersion should be old
enough that they will not be used in a Dalmatian deployment:

- Ansible 2.11
- Docker API 1.42, included since Docker engine 23.0.0

Change-Id: Ie315004715a1cb5a91dd54bc64b0a8fd0af650ec
This commit is contained in:
leiyashuai 2022-09-12 15:56:04 +08:00 committed by Mark Goddard
parent 4615f247fc
commit b3a66ef3d4
4 changed files with 54 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
ansible/library/kolla_toolbox.py
View File

@ -12,11 +12,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from distutils.version import StrictVersion
import json import json
import re import re
from ansible.module_utils.ansible_release import __version__ as ansible_version
from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.basic import AnsibleModule
from ast import literal_eval from ast import literal_eval
@ -117,17 +115,14 @@ def gen_commandline(params):
if params.get('module_name'): if params.get('module_name'):
command.extend(['-m', params.get('module_name')]) command.extend(['-m', params.get('module_name')])
if params.get('module_args'): if params.get('module_args'):
if StrictVersion(ansible_version) < StrictVersion('2.11.0'): try:
module_args = params.get('module_args') module_args = literal_eval(params.get('module_args'))
else: except SyntaxError:
try: if not isinstance(params.get('module_args'), str):
module_args = literal_eval(params.get('module_args')) raise
except SyntaxError:
if not isinstance(params.get('module_args'), str):
raise
# account for string arguments # account for string arguments
module_args = split(params.get('module_args')) module_args = split(params.get('module_args'))
if isinstance(module_args, dict): if isinstance(module_args, dict):
module_args = ' '.join("{}='{}'".format(key, value) module_args = ' '.join("{}='{}'".format(key, value)
for key, value in module_args.items()) for key, value in module_args.items())
@ -147,11 +142,6 @@ def get_docker_client():
return docker.APIClient return docker.APIClient
def docker_supports_environment_in_exec(client):
docker_version = StrictVersion(client.api_version)
return docker_version >= StrictVersion('1.25')
def use_docker(module): def use_docker(module):
client = get_docker_client()( client = get_docker_client()(
version=module.params.get('api_version'), version=module.params.get('api_version'),
@ -167,79 +157,43 @@ def use_docker(module):
if 'user' in module.params: if 'user' in module.params:
kwargs['user'] = module.params['user'] kwargs['user'] = module.params['user']
# NOTE(mgoddard): Docker 1.12 has API version 1.24, and was installed by # Use the JSON output formatter, so that we can parse it.
# kolla-ansible bootstrap-servers on Rocky and earlier releases. This API environment = {"ANSIBLE_STDOUT_CALLBACK": "json",
# version does not have support for specifying environment variables for "ANSIBLE_LOAD_CALLBACK_PLUGINS": "True"}
# exec jobs, which is necessary to use the Ansible JSON output formatter. job = client.exec_create(kolla_toolbox, command_line,
# While we continue to support this version of Docker, fall back to the old environment=environment, **kwargs)
# regex-based method for API version 1.24 and earlier. json_output = client.exec_start(job)
# TODO(mgoddard): Remove this conditional (keep the if) when we require
# Docker API version 1.25+.
if docker_supports_environment_in_exec(client):
# Use the JSON output formatter, so that we can parse it.
environment = {"ANSIBLE_STDOUT_CALLBACK": "json",
"ANSIBLE_LOAD_CALLBACK_PLUGINS": "True"}
job = client.exec_create(kolla_toolbox, command_line,
environment=environment, **kwargs)
json_output = client.exec_start(job)
try: try:
output = json.loads(json_output) output = json.loads(json_output)
except Exception: except Exception:
module.fail_json( module.fail_json(
msg='Can not parse the inner module output: %s' % json_output) msg='Can not parse the inner module output: %s' % json_output)
# Expected format is the following: # Expected format is the following:
# { # {
# "plays": [ # "plays": [
# { # {
# "tasks": [ # "tasks": [
# { # {
# "hosts": { # "hosts": {
# "localhost": { # "localhost": {
# <module result> # <module result>
# } # }
# } # }
# } # }
# ] # ]
# { # {
# ] # ]
# } # }
try: try:
ret = output['plays'][0]['tasks'][0]['hosts']['localhost'] ret = output['plays'][0]['tasks'][0]['hosts']['localhost']
except (KeyError, IndexError): except (KeyError, IndexError):
module.fail_json( module.fail_json(
msg='Ansible JSON output has unexpected format: %s' % output) msg='Ansible JSON output has unexpected format: %s' % output)
# Remove Ansible's internal variables from returned fields.
ret.pop('_ansible_no_log', None)
else:
job = client.exec_create(kolla_toolbox, command_line, **kwargs)
output = client.exec_start(job)
for exp in [JSON_REG, NON_JSON_REG]:
m = exp.match(output)
if m:
inner_output = m.groupdict().get('stdout')
status = m.groupdict().get('status')
break
else:
module.fail_json(
msg='Can not parse the inner module output: %s' % output)
ret = dict()
try:
ret = json.loads(inner_output)
except ValueError:
# Some modules (e.g. command) do not produce a JSON output.
# Instead, check the status, and assume changed on success.
ret['stdout'] = inner_output
if status != "SUCCESS":
ret['failed'] = True
else:
# No way to know whether changed - assume yes.
ret['changed'] = True
# Remove Ansible's internal variables from returned fields.
ret.pop('_ansible_no_log', None)
return ret return ret

3
ansible/module_utils/kolla_container_worker.py
View File

@ -28,7 +28,6 @@ class ContainerWorker(ABC):
self.changed = False self.changed = False
# Use this to store arguments to pass to exit_json(). # Use this to store arguments to pass to exit_json().
self.result = {} self.result = {}
self._cgroupns_mode_supported = True
self.systemd = SystemdWorker(self.params) self.systemd = SystemdWorker(self.params)
@ -141,8 +140,6 @@ class ContainerWorker(ABC):
pass pass
def compare_cgroupns_mode(self, container_info): def compare_cgroupns_mode(self, container_info):
if not self._cgroupns_mode_supported:
return False
new_cgroupns_mode = self.params.get('cgroupns_mode') new_cgroupns_mode = self.params.get('cgroupns_mode')
if new_cgroupns_mode is None: if new_cgroupns_mode is None:
# means we don't care what it is # means we don't care what it is

22
ansible/module_utils/kolla_docker_worker.py
View File

@ -19,8 +19,6 @@ import os
from ansible.module_utils.kolla_container_worker import COMPARE_CONFIG_CMD from ansible.module_utils.kolla_container_worker import COMPARE_CONFIG_CMD
from ansible.module_utils.kolla_container_worker import ContainerWorker from ansible.module_utils.kolla_container_worker import ContainerWorker
from distutils.version import StrictVersion
def get_docker_client(): def get_docker_client():
return docker.APIClient return docker.APIClient
@ -38,13 +36,8 @@ class DockerWorker(ContainerWorker):
self.dc = get_docker_client()(**options) self.dc = get_docker_client()(**options)
self._cgroupns_mode_supported = ( self._dimensions_kernel_memory_removed = True
StrictVersion(self.dc._version) >= StrictVersion('1.41')) self.dimension_map.pop('kernel_memory', None)
self._dimensions_kernel_memory_removed = (
StrictVersion(self.dc._version) >= StrictVersion('1.42'))
if self._dimensions_kernel_memory_removed:
self.dimension_map.pop('kernel_memory', None)
def generate_tls(self): def generate_tls(self):
tls = {'verify': self.params.get('tls_verify')} tls = {'verify': self.params.get('tls_verify')}
@ -304,12 +297,11 @@ class DockerWorker(ContainerWorker):
host_config = self.dc.create_host_config(**options) host_config = self.dc.create_host_config(**options)
if self._cgroupns_mode_supported: # NOTE(yoctozepto): python-docker does not support CgroupnsMode
# NOTE(yoctozepto): python-docker does not support CgroupnsMode # natively so we stuff it in manually.
# natively so we stuff it in manually. cgroupns_mode = self.params.get('cgroupns_mode')
cgroupns_mode = self.params.get('cgroupns_mode') if cgroupns_mode is not None:
if cgroupns_mode is not None: host_config['CgroupnsMode'] = cgroupns_mode
host_config['CgroupnsMode'] = cgroupns_mode
# detached containers should only log to journald # detached containers should only log to journald
if self.params.get('detach'): if self.params.get('detach'):

45
tests/kolla_container_tests/test_docker_worker.py
View File

@ -244,22 +244,8 @@ class TestMainModule(base.BaseTestCase):
result=False, result=False,
some_key="some_value") some_key="some_value")
def test_sets_cgroupns_mode_supported_false(self):
self.dw = get_DockerWorker(self.fake_data['params'])
self.assertFalse(self.dw._cgroupns_mode_supported)
def test_sets_cgroupns_mode_supported_true(self):
self.dw = get_DockerWorker(self.fake_data['params'],
docker_api_version='1.41')
self.assertTrue(self.dw._cgroupns_mode_supported)
def test_sets_dimensions_kernelmemory_supported_true(self):
self.dw = get_DockerWorker(self.fake_data['params'])
self.assertFalse(self.dw._dimensions_kernel_memory_removed)
def test_sets_dimensions_kernelmemory_supported_false(self): def test_sets_dimensions_kernelmemory_supported_false(self):
self.dw = get_DockerWorker(self.fake_data['params'], self.dw = get_DockerWorker(self.fake_data['params'])
docker_api_version='1.42')
self.assertTrue(self.dw._dimensions_kernel_memory_removed) self.assertTrue(self.dw._dimensions_kernel_memory_removed)
def test_common_options_defaults(self): def test_common_options_defaults(self):
@ -1416,10 +1402,10 @@ class TestAttrComp(base.BaseTestCase):
self.dw.module.fail_json.assert_not_called() self.dw.module.fail_json.assert_not_called()
self.assertFalse(resp) self.assertFalse(resp)
def test_compare_dimensions_key_no_more_supported(self): def test_compare_dimensions_invalid_unit(self):
self.fake_data['params']['dimensions'] = { self.fake_data['params']['dimensions'] = {
'mem_limit': '1024', 'mem_reservation': '3M', 'mem_limit': '1024', 'mem_reservation': '3M',
'memswap_limit': '2g', 'kernel_memory': '4M'} 'memswap_limit': '2E'}
container_info = dict() container_info = dict()
container_info['HostConfig'] = { container_info['HostConfig'] = {
'CpuPeriod': 0, 'Memory': 1024, 'CpuQuota': 0, 'CpuPeriod': 0, 'Memory': 1024, 'CpuQuota': 0,
@ -1428,26 +1414,8 @@ class TestAttrComp(base.BaseTestCase):
'MemoryReservation': 3 * 1024 * 1024, 'Ulimits': []} 'MemoryReservation': 3 * 1024 * 1024, 'Ulimits': []}
self.dw = get_DockerWorker(self.fake_data['params']) self.dw = get_DockerWorker(self.fake_data['params'])
self.dw.compare_dimensions(container_info) self.dw.compare_dimensions(container_info)
expected_msg = ("The dimension [kernel_memory] is no more "
"supported by Docker, please remove it from "
"yours configs or change to the new one.")
self.dw.module.fail_json.assert_called_once_with(
failed=True, msg=expected_msg)
def test_compare_dimensions_invalid_unit(self):
self.fake_data['params']['dimensions'] = {
'mem_limit': '1024', 'mem_reservation': '3M',
'memswap_limit': '2g', 'kernel_memory': '4E'}
container_info = dict()
container_info['HostConfig'] = {
'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 1024, 'CpuQuota': 0,
'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
'CpusetMems': '', 'MemorySwap': 2 * 1024 * 1024 * 1024,
'MemoryReservation': 3 * 1024 * 1024, 'Ulimits': []}
self.dw = get_DockerWorker(self.fake_data['params'])
self.dw.compare_dimensions(container_info)
expected_msg = ("The docker dimension unit [e] is " expected_msg = ("The docker dimension unit [e] is "
"not supported for the dimension [4E]." "not supported for the dimension [2E]."
" The currently supported units are " " The currently supported units are "
"['b', 'k', 'm', 'g'].") "['b', 'k', 'm', 'g'].")
self.dw.module.fail_json.assert_called_once_with( self.dw.module.fail_json.assert_called_once_with(
@ -1538,7 +1506,7 @@ class TestAttrComp(base.BaseTestCase):
self.dw = get_DockerWorker(self.fake_data['params']) self.dw = get_DockerWorker(self.fake_data['params'])
self.assertFalse(self.dw.compare_dimensions(container_info)) self.assertFalse(self.dw.compare_dimensions(container_info))
def test_compare_dimensions_kernel_memory_1_42(self): def test_compare_dimensions_kernel_memory_unsupported(self):
self.fake_data['params']['dimensions'] = { self.fake_data['params']['dimensions'] = {
'kernel_memory': '1024'} 'kernel_memory': '1024'}
container_info = dict() container_info = dict()
@ -1547,8 +1515,7 @@ class TestAttrComp(base.BaseTestCase):
'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0, 'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0, 'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
'Ulimits': []} 'Ulimits': []}
self.dw = get_DockerWorker(self.fake_data['params'], self.dw = get_DockerWorker(self.fake_data['params'])
docker_api_version='1.42')
self.dw.compare_dimensions(container_info) self.dw.compare_dimensions(container_info)
self.dw.module.exit_json.assert_called_once_with( self.dw.module.exit_json.assert_called_once_with(
failed=True, msg=repr("Unsupported dimensions"), failed=True, msg=repr("Unsupported dimensions"),