NSX|V: fix timeout out issues

The patch addresses 2 things:
1. The deploymen of edges may take longer that the configured timeout.
   This opertaion now has its own timeout (hardcoded to 20 minutes).
2. The configured timout is now bumoped to double. We have seen that
   underload there are opertaions that have taken longer than the
   defult 2 minutes

Change-Id: I3ae519b84be58c0f8044fd283aba45f2ed53e431
This commit is contained in:
Gary Kotton 2017-11-09 03:35:41 -08:00
parent 219be1f34b
commit cb5fcafe18
3 changed files with 14 additions and 7 deletions

View File

@ -693,7 +693,7 @@ nsxv_opts = [
"networks, as well as ranges of VLAN tags on each " "networks, as well as ranges of VLAN tags on each "
"available for allocation to networks.")), "available for allocation to networks.")),
cfg.IntOpt('nsx_transaction_timeout', cfg.IntOpt('nsx_transaction_timeout',
default=120, default=240,
help=_("Timeout interval for NSX backend transactions.")), help=_("Timeout interval for NSX backend transactions.")),
cfg.BoolOpt('share_edges_between_tenants', cfg.BoolOpt('share_edges_between_tenants',
default=True, default=True,

View File

@ -135,8 +135,10 @@ class VcnsApiHelper(object):
return ctx.__dict__.get('request_id') return ctx.__dict__.get('request_id')
def request(self, method, uri, params=None, headers=None, def request(self, method, uri, params=None, headers=None,
encodeparams=True): encodeparams=True, timeout=None):
uri = self.address + uri uri = self.address + uri
if timeout is None:
timeout = self.timeout
if headers is None: if headers is None:
headers = {} headers = {}
@ -161,7 +163,7 @@ class VcnsApiHelper(object):
verify=self.verify_cert, verify=self.verify_cert,
data=data, data=data,
headers=headers, headers=headers,
timeout=self.timeout) timeout=timeout)
except requests.exceptions.Timeout: except requests.exceptions.Timeout:
raise exceptions.ResourceTimedOut(uri=uri) raise exceptions.ResourceTimedOut(uri=uri)

View File

@ -88,6 +88,7 @@ NETWORK_TYPES = ['Network', 'VirtualWire', 'DistributedVirtualPortgroup']
ROUTING_CONFIG = "routing/config" ROUTING_CONFIG = "routing/config"
BGP_ROUTING_CONFIG = "routing/config/bgp" BGP_ROUTING_CONFIG = "routing/config/bgp"
ELAPSED_TIME_THRESHOLD = 30 ELAPSED_TIME_THRESHOLD = 30
MAX_EDGE_DEPLOY_TIMEOUT = 1200
def retry_upon_exception_exclude_error_codes( def retry_upon_exception_exclude_error_codes(
@ -123,8 +124,9 @@ class Vcns(object):
@retry_upon_exception(exceptions.ServiceConflict) @retry_upon_exception(exceptions.ServiceConflict)
def _client_request(self, client, method, uri, def _client_request(self, client, method, uri,
params, headers, encodeParams): params, headers, encodeParams, timeout=None):
return client(method, uri, params, headers, encodeParams) return client(method, uri, params, headers, encodeParams,
timeout=timeout)
def do_request(self, method, uri, params=None, format='json', **kwargs): def do_request(self, method, uri, params=None, format='json', **kwargs):
msg = ("VcnsApiHelper('%(method)s', '%(uri)s', '%(body)s')" % msg = ("VcnsApiHelper('%(method)s', '%(uri)s', '%(body)s')" %
@ -140,9 +142,11 @@ class Vcns(object):
else: else:
_client = self.xmlapi_client.request _client = self.xmlapi_client.request
timeout = kwargs.get('timeout')
ts = time.time() ts = time.time()
header, content = self._client_request(_client, method, uri, params, header, content = self._client_request(_client, method, uri, params,
headers, encodeParams) headers, encodeParams,
timeout=timeout)
te = time.time() te = time.time()
elapsed_time = te - ts elapsed_time = te - ts
@ -170,7 +174,8 @@ class Vcns(object):
@retry_upon_exception(exceptions.RequestBad) @retry_upon_exception(exceptions.RequestBad)
def deploy_edge(self, request): def deploy_edge(self, request):
uri = URI_PREFIX uri = URI_PREFIX
return self.do_request(HTTP_POST, uri, request, decode=False) return self.do_request(HTTP_POST, uri, request, decode=False,
timeout=MAX_EDGE_DEPLOY_TIMEOUT)
def update_edge(self, edge_id, request): def update_edge(self, edge_id, request):
uri = "%s/%s" % (URI_PREFIX, edge_id) uri = "%s/%s" % (URI_PREFIX, edge_id)