openstack/openstacksdk
Code Issues Proposed changes

Allow resources to be skipped on project cleanup

Browse Source 
Story: 2010370
Task: 46596
Change-Id: Id6d68e40656c92910f491fe3b66e40c69b44e352
Signed-off-by: Jan Hartkopf <jhartkopf@inovex.de>
This commit is contained in:
Jan Hartkopf 2023-03-27 15:46:36 +02:00
parent e7ee461d9b
commit 068cc997d5
10 changed files with 303 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
openstack/block_storage/v3/_proxy.py
View File

@ -1675,6 +1675,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
identified_resources=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
# It is not possible to delete backup if there are dependent backups.
# In order to be able to do cleanup those is required to have multiple
@ -1684,6 +1685,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
# required to limit amount of iterations we do (currently pick 10). In
# dry_run all those iterations are doing not what we want, therefore
# only iterate in a real cleanup mode.
if not self.should_skip_resource_cleanup("backup", skip_resources):
if dry_run:
# Just iterate and evaluate backups in dry_run mode
for obj in self.backups(details=False):
@ -1735,6 +1737,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
# Well, did our best, still try further
pass
if not self.should_skip_resource_cleanup("snapshot", skip_resources):
snapshots = []
for obj in self.snapshots(details=False):
need_delete = self._service_cleanup_del_res(
@ -1757,6 +1760,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
# Well, did our best, still try further
pass
if not self.should_skip_resource_cleanup("volume", skip_resources):
for obj in self.volumes(details=True):
self._service_cleanup_del_res(
self.delete_volume,

4
openstack/cloud/openstackcloud.py
View File

@ -813,6 +813,7 @@ class _OpenStackCloudMixin:
status_queue=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
"""Cleanup the project resources.
@ -829,6 +830,8 @@ class _OpenStackCloudMixin:
:param resource_evaluation_fn: A callback function, which will be
invoked for each resurce and must return True/False depending on
whether resource need to be deleted or not.
:param skip_resources: List of specific resources whose cleanup should
be skipped.
"""
dependencies = {}
get_dep_fn_name = '_get_cleanup_dependencies'
@ -879,6 +882,7 @@ class _OpenStackCloudMixin:
identified_resources=cleanup_resources,
filters=filters,
resource_evaluation_fn=resource_evaluation_fn,
skip_resources=skip_resources,
)
except exceptions.ServiceDisabledException:
# same reason as above

4
openstack/compute/v2/_proxy.py
View File

@ -2548,7 +2548,11 @@ class Proxy(proxy.Proxy):
identified_resources=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
if self.should_skip_resource_cleanup("server", skip_resources):
return
servers = []
for obj in self.servers():
need_delete = self._service_cleanup_del_res(

6
openstack/dns/v2/_proxy.py
View File

@ -667,7 +667,9 @@ class Proxy(proxy.Proxy):
identified_resources=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
if not self.should_skip_resource_cleanup("zone", skip_resources):
# Delete all zones
for obj in self.zones():
self._service_cleanup_del_res(
@ -679,6 +681,10 @@ class Proxy(proxy.Proxy):
filters=filters,
resource_evaluation_fn=resource_evaluation_fn,
)
if not self.should_skip_resource_cleanup(
"floating_ip", skip_resources
):
# Unset all floatingIPs
# NOTE: FloatingIPs are not cleaned when filters are set
for obj in self.floating_ips():

32
openstack/network/v2/_proxy.py
View File

@ -6212,8 +6212,13 @@ class Proxy(proxy.Proxy, Generic[T]):
identified_resources=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
project_id = self.get_project_id()
if not self.should_skip_resource_cleanup(
"floating_ip", skip_resources
):
# Delete floating_ips in the project if no filters defined OR all
# filters are matching and port_id is empty
for obj in self.ips(project_id=project_id):
@ -6227,6 +6232,9 @@ class Proxy(proxy.Proxy, Generic[T]):
resource_evaluation_fn=fip_cleanup_evaluation,
)
if not self.should_skip_resource_cleanup(
"security_group", skip_resources
):
# Delete (try to delete) all security groups in the project
# Let's hope we can't drop SG in use
for obj in self.security_groups(project_id=project_id):
@ -6241,12 +6249,19 @@ class Proxy(proxy.Proxy, Generic[T]):
resource_evaluation_fn=resource_evaluation_fn,
)
if (
not self.should_skip_resource_cleanup("network", skip_resources)
and not self.should_skip_resource_cleanup("port", skip_resources)
and not self.should_skip_resource_cleanup("subnet", skip_resources)
):
# Networks are crazy, try to delete router+net+subnet
# if there are no "other" ports allocated on the net
for net in self.networks(project_id=project_id):
network_has_ports_allocated = False
router_if = list()
for port in self.ports(project_id=project_id, network_id=net.id):
for port in self.ports(
project_id=project_id, network_id=net.id
):
self.log.debug('Looking at port %s' % port)
if port.device_owner in [
'network:router_interface',
@ -6307,7 +6322,9 @@ class Proxy(proxy.Proxy, Generic[T]):
resource_evaluation_fn=None,
)
# Drop ports not belonging to anybody
for port in self.ports(project_id=project_id, network_id=net.id):
for port in self.ports(
project_id=project_id, network_id=net.id
):
if port.device_owner is None or port.device_owner == '':
self._service_cleanup_del_res(
self.delete_port,
@ -6320,7 +6337,9 @@ class Proxy(proxy.Proxy, Generic[T]):
)
# Drop all subnets in the net (no further conditions)
for obj in self.subnets(project_id=project_id, network_id=net.id):
for obj in self.subnets(
project_id=project_id, network_id=net.id
):
self._service_cleanup_del_res(
self.delete_subnet,
obj,
@ -6342,7 +6361,14 @@ class Proxy(proxy.Proxy, Generic[T]):
filters=None,
resource_evaluation_fn=None,
)
else:
self.log.debug(
"Skipping cleanup of networks, ports and subnets "
"as those resources require none of them to be "
"excluded, but at least one should be kept"
)
if not self.should_skip_resource_cleanup("router", skip_resources):
# It might happen, that we have routers not attached to anything
for obj in self.routers():
ports = list(self.ports(device_id=obj.id))

6
openstack/object_store/v1/_proxy.py
View File

@ -1130,7 +1130,13 @@ class Proxy(proxy.Proxy):
identified_resources=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
if self.should_skip_resource_cleanup(
"container", skip_resources
) or self.should_skip_resource_cleanup("object", skip_resources):
return
is_bulk_delete_supported = False
bulk_delete_max_per_request = None
try:

4
openstack/orchestration/v1/_proxy.py
View File

@ -549,7 +549,11 @@ class Proxy(proxy.Proxy):
identified_resources=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
if self.should_skip_resource_cleanup("stack", skip_resources):
return
stacks = []
for obj in self.stacks():
need_delete = self._service_cleanup_del_res(

27
openstack/proxy.py
View File

@ -100,7 +100,7 @@ class Proxy(adapter.Adapter, Generic[T]):
influxdb_config=None,
influxdb_client=None,
*args,
**kwargs
**kwargs,
):
# NOTE(dtantsur): keystoneauth defaults retriable_status_codes to None,
# override it with a class-level value.
@ -144,7 +144,7 @@ class Proxy(adapter.Adapter, Generic[T]):
connect_retries=1,
global_request_id=None,
*args,
**kwargs
**kwargs,
):
conn = self._get_connection()
if not global_request_id:
@ -180,7 +180,7 @@ class Proxy(adapter.Adapter, Generic[T]):
connect_retries=connect_retries,
raise_exc=raise_exc,
global_request_id=global_request_id,
**kwargs
**kwargs,
),
),
expiration_time=expiration_time,
@ -196,7 +196,7 @@ class Proxy(adapter.Adapter, Generic[T]):
connect_retries=connect_retries,
raise_exc=raise_exc,
global_request_id=global_request_id,
**kwargs
**kwargs,
)
for h in response.history:
@ -623,7 +623,7 @@ class Proxy(adapter.Adapter, Generic[T]):
requires_id=True,
base_path=None,
skip_cache=False,
**attrs
**attrs,
):
"""Fetch a resource
@ -665,7 +665,7 @@ class Proxy(adapter.Adapter, Generic[T]):
paginated=True,
base_path=None,
jmespath_filters=None,
**attrs
**attrs,
) -> Generator[T, None, None]:
"""List a resource
@ -736,6 +736,7 @@ class Proxy(adapter.Adapter, Generic[T]):
identified_resources=None,
filters=None,
resource_evaluation_fn=None,
skip_resources=None,
):
return None
@ -814,6 +815,20 @@ class Proxy(adapter.Adapter, Generic[T]):
else:
return False
def should_skip_resource_cleanup(self, resource=None, skip_resources=None):
if resource is None or skip_resources is None:
return False
resource_name = f"{self.service_type.replace('-', '_')}.{resource}"
if resource_name in skip_resources:
self.log.debug(
f"Skipping resource {resource_name} " "in project cleanup"
)
return True
return False
# TODO(stephenfin): Remove this and all users. Use of this generally indicates
# a missing Resource type.

10
openstack/tests/unit/test_proxy.py
View File

@ -728,6 +728,7 @@ class TestProxyCleanup(base.TestCase):
self.res_no_updated.created_at = '2020-01-02T03:04:05'
self.sot = proxy.Proxy(self.session)
self.sot.service_type = "block-storage"
self.delete_mock = mock.Mock()
@ -867,3 +868,12 @@ class TestProxyCleanup(base.TestCase):
)
)
self.assertEqual(self.res, q.get_nowait())
def test_should_skip_resource_cleanup(self):
excluded = ["block_storage.backup"]
self.assertTrue(
self.sot.should_skip_resource_cleanup("backup", excluded)
)
self.assertFalse(
self.sot.should_skip_resource_cleanup("volume", excluded)
)

6
releasenotes/notes/project-cleanup-exclude-option-65cba962eaa5b61a.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- |
Project cleanup now supports skipping specific resources,
which will be kept as-is. Resource names are based on the
resource registry names, e. g. "block_storage.volume".