diff --git a/orm/base_config.py b/orm/base_config.py index 1fc04b2b..4389dbae 100644 --- a/orm/base_config.py +++ b/orm/base_config.py @@ -12,8 +12,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import os +from os.path import join from oslo_config import cfg +from ConfigParser import ConfigParser + CONF = cfg.CONF @@ -140,6 +144,16 @@ OrmCmsGroup = [ CONF.register_group(orm_cms_group) CONF.register_opts(OrmCmsGroup, orm_cms_group) +config = ConfigParser() + +if CONF.config_file: + cfgfile = CONF.config_file[0] +else: + workspace = os.environ['PWD'] + cfgfile = join(workspace, 'etc', 'ranger', 'ranger.conf') + +config.read(cfgfile) +default_flavor = 'xx' # fms config options in [fms] group orm_fms_group = cfg.OptGroup(name='fms', title='Orm Fms Options') @@ -150,12 +164,62 @@ OrmFmsGroup = [ help='Fms port.'), cfg.StrOpt('log', default='fms.log', - help='Fms log name.') + help='Fms log name.'), + cfg.ListOpt('flavor_series', + default=[default_flavor], + help='Supported flavor series.'), + cfg.IntOpt('flavor_swap_file_limit', + default=0, + help='Flavor series swap file size limit.'), + cfg.IntOpt('flavor_ephemeral_limit', + default=0, + help='Flavor series ephemeral limit.') ] CONF.register_group(orm_fms_group) CONF.register_opts(OrmFmsGroup, orm_fms_group) +autogen_es = set() + +flavor_group = cfg.OptGroup(name='flavor_series_metadata', + title='A Group of Flavor Series') +CONF.register_group(flavor_group) + + +if default_flavor in CONF.fms.flavor_series: + default_dict = {'alt_vcpu_limit': 1, 'alt_vram_limit': 1024} + + FlavorGroup = [ + cfg.DictOpt(default_flavor, + default=default_dict, + help="Dict that contains default flavor series metadata.") + ] + CONF.register_opts(FlavorGroup, flavor_group) +else: + sections = config.sections() + series_metadata = [s for s in sections if "_flavor_series_metadata" in s] + for metadata in series_metadata: + flavor = metadata.replace('_flavor_series_metadata', '') + flavor_dict = dict(set(config.items(metadata)) - + set(config.items('DEFAULT'))) + + FlavorGroup = [ + cfg.DictOpt(flavor, + default=flavor_dict, + help="Dict that contains flavor series metadata. ") + ] + CONF.register_opts(FlavorGroup, flavor_group) + + for key, value in flavor_dict.items(): + if key.startswith("es_"): + autogen_es.add(value.split(': ')[0]) + +AutogenEsGroup = [ + cfg.ListOpt('autogen_extra_specs', + default=list(autogen_es), + help="List of auto generated extra specs.") +] +CONF.register_opts(AutogenEsGroup, flavor_group) # audit config options in [audit] group orm_audit_group = cfg.OptGroup(name='audit', title='Orm Audit Options') @@ -266,7 +330,8 @@ conn = CONF.database.connection db_connect = conn.replace("mysql+pymysql", "mysql") if conn else None ssl_verify = CONF.ssl_verify -token_auth_version = '3' if (CONF.keystone_authtoken.auth_version == 'v3') else '2.0' +token_auth_version = '3' if (CONF.keystone_authtoken.auth_version == + 'v3') else '2.0' cert_path = CONF.ranger_agent_client_cert_path https_enabled = CONF.ranger_agent_https_enabled @@ -333,7 +398,8 @@ def server_request_auth(server_name): # authentication settings request_authentication = { "enabled": CONF.keystone_authtoken.auth_enabled, - # The Keystone version currently in use. For Ranger, use '3' by default. + # The Keystone version currently in use. For Ranger, + # use '3' by default. "keystone_version": token_auth_version, "mech_id": CONF.keystone_authtoken.username, "mech_pass": CONF.keystone_authtoken.password, @@ -353,8 +419,9 @@ def server_request_auth(server_name): def get_log_config(log_file_name, ranger_service, ranger_service_module): - # Ranger logging template - we want to have the option of not routing to logfiles - # for all loggers except 'pecan' and 'py.warnings', which only logs to console + # Ranger logging template - we want to have the option of not + # routing to logfiles for all loggers except 'pecan' and + # 'py.warnings', which only logs to console logging_template = { 'root': {'level': 'INFO', 'handlers': handler_list}, 'loggers': { @@ -405,8 +472,9 @@ def get_log_config(log_file_name, ranger_service, ranger_service_module): }, 'color': { '()': 'pecan.log.ColorFormatter', - 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' - '[%(threadName)s] %(message)s'), + 'format': ( + '%(asctime)s [%(padded_color_levelname)s] [%(name)s]' + '[%(threadName)s] %(message)s'), '__force_dict__': True } } diff --git a/orm/orm_client/ormcli/__init__.py b/orm/orm_client/ormcli/__init__.py index e69de29b..1ebae3dc 100644 --- a/orm/orm_client/ormcli/__init__.py +++ b/orm/orm_client/ormcli/__init__.py @@ -0,0 +1,2 @@ +from oslo_config import cfg +cfg.CONF([], project='ranger', validate_default_values=True) diff --git a/orm/orm_client/ormcli/fmscli.py b/orm/orm_client/ormcli/fmscli.py index f9155522..9a171cbb 100644 --- a/orm/orm_client/ormcli/fmscli.py +++ b/orm/orm_client/ormcli/fmscli.py @@ -161,7 +161,7 @@ def add_to_parser(service_sub): h1, h2 = ('[<"X-RANGER-Client" header>]', '[--visibility ] [--region ] [--tenant ' - '] [--series {gv,nv,ns,nd,ss}] [--alias ] ' + '] [--series ] [--alias ] ' '[--starts_with ] [--contains ] ' '[--vm_type ] [--vnf_name ]') parser_list_flavor = subparsers.add_parser('list_flavors', @@ -175,8 +175,7 @@ def add_to_parser(service_sub): help='flavor name starts with *') parser_list_flavor.add_argument('--contains', type=str, help='* contains in flavor name') - parser_list_flavor.add_argument('--series', type=str, - choices=['gv', 'nv', 'ns', 'nd', 'ss']) + parser_list_flavor.add_argument('--series', type=str, help='series name') parser_list_flavor.add_argument('--alias', type=str, help='flavor alias') parser_list_flavor.add_argument('--vm_type', type=str, help='vm type') parser_list_flavor.add_argument('--vnf_name', type=str, help='vnf name') @@ -342,7 +341,11 @@ def get_token(timeout, args, host): 'Failed in get_token, host: {}, region: {}'.format(host, auth_region)) url = url % (keystone_ep,) - data = data % (base_config.user_domain_name, username, password, tenant_name, base_config.project_domain_name,) + data = data % (base_config.user_domain_name, + username, + password, + tenant_name, + base_config.project_domain_name,) if args.verbose: print( @@ -370,8 +373,11 @@ def get_environment_variable(argument): def run(args): - rms_url = args.rms_base_url if args.rms_base_url else base_config.rms['base_url'] - host = args.fms_base_url if args.fms_base_url else base_config.fms['base_url'] + rms_url = args.rms_base_url if args.rms_base_url else \ + base_config.rms['base_url'] + host = args.fms_base_url if args.fms_base_url else \ + base_config.fms['base_url'] + port = args.port if args.port else base_config.fms['port'] data = args.datafile.read() if 'datafile' in args else '{}' timeout = args.timeout if args.timeout else 10 diff --git a/orm/services/db_setup.py b/orm/services/db_setup.py index 76c6374d..8763a068 100644 --- a/orm/services/db_setup.py +++ b/orm/services/db_setup.py @@ -63,6 +63,8 @@ def main(argv=None): '/orm/services/customer_manager/scripts/db_scripts/ranger_cms_update_db.sql', CONF.ranger_base + '/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql', + CONF.ranger_base + + '/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_update_db.sql', CONF.ranger_base + '/orm/services/image_manager/scripts/db_scripts/create_db.sql' ] diff --git a/orm/services/flavor_manager/config.py b/orm/services/flavor_manager/config.py index 2aca3026..ec5e9966 100755 --- a/orm/services/flavor_manager/config.py +++ b/orm/services/flavor_manager/config.py @@ -31,22 +31,6 @@ database = { 'connection_string': config.db_connect } -# this table is for calculating default extra specs needed -extra_spec_needed_table = { - "p1": { - "aggregate_instance_extra_specs____p1": "true", - "hw____mem_page_size": "large" - } -} - -# any key will be added to extra_spec_needed_table need to be added here -default_extra_spec_calculated_table = { - "aggregate_instance_extra_specs____p1": "", - "hw____mem_page_size": "", - "hw____cpu_policy": "", - "hw____numa_nodes": "" -} - application_root = 'http://localhost:{0}'.format(server['port']) api = { @@ -72,39 +56,6 @@ api = { } -flavor_series = { - 'valid_series': [ - 'p1' - ] -} - -# valid_flavor_options -flavor_options = { - 'valid_p1_numa_value': 'n0', - 'valid_p1_opt_values': [ - 'n0', 'i2' - ] -} - -flavor_limits = { - # All flavor limits will be converted to integers, and must not be non-numeric. - # Root disk, block storage and object storage don't have set limits - # vcpu_limit and ephemeral_limit values are in GB - # vram_limit and swap_file_limit values are in MB and must be a multiple of 1024 - - "swap_file_limit": "327680", - "ephemeral_limit": "2000", - - # for 'p1' series: - # vcpu_limit and vram_limit for 'p1' series with "n0":"true" option - "p1_n0_vcpu_limit": "80", - "p1_n0_vram_limit": "327680", - # vcpu_limit and vram_limit for 'p1' series with "n0":"false" option - "p1_nx_vcpu_limit": "40", - "p1_nx_vram_limit": "163840" - -} - verify = config.CONF.ssl_verify authentication = config.server_request_auth(server['name']) diff --git a/orm/services/flavor_manager/fms_rest/data/sql_alchemy/db_models.py b/orm/services/flavor_manager/fms_rest/data/sql_alchemy/db_models.py index e653d9ee..6a11142a 100755 --- a/orm/services/flavor_manager/fms_rest/data/sql_alchemy/db_models.py +++ b/orm/services/flavor_manager/fms_rest/data/sql_alchemy/db_models.py @@ -3,8 +3,8 @@ from __builtin__ import reversed from orm.services.flavor_manager.fms_rest.logger import get_logger from orm.services.flavor_manager.fms_rest.logic.error_base import ErrorStatus +from oslo_config import cfg from oslo_db.sqlalchemy import models -from pecan import conf from sqlalchemy import BigInteger, Column, ForeignKey, Integer, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship, validates @@ -32,8 +32,9 @@ class FMSBaseModel(models.ModelBase): class Flavor(Base, FMSBaseModel): - '''Flavor is a DataObject contains all the fields defined in Flavor table record. - defined as SqlAlchemy model map to a table + '''Flavor is a DataObject contains all the fields defined + 'in Flavor table record. + 'defined as SqlAlchemy model map to a table ''' __tablename__ = "flavor" @@ -61,10 +62,11 @@ class Flavor(Base, FMSBaseModel): cascade="all, delete, delete-orphan") def __repr__(self): - text = "Flavor(internal_id={}, id={}, name={}, alias={}, description={}, " \ - "series={}, ram={}, vcpus={}, disk={}, swap={}, ephemeral={}," \ - "visibility={}, flavor_extra_specs={}, flavor_tags={}," \ - "flavor_options={}, flavor_regions={}, flavor_tenants={})". \ + text = "Flavor(internal_id={}, id={}, name={}, alias={}, " \ + "description={}, series={}, ram={}, vcpus={}, disk={}, " \ + "swap={}, ephemeral={}, visibility={}, " \ + "flavor_extra_specs={}, flavor_tags={}, flavor_options={}, " \ + "flavor_regions={}, flavor_tenants={})". \ format(self.internal_id, self.id, self.name, @@ -124,9 +126,10 @@ class Flavor(Base, FMSBaseModel): @validates("series") def validate_series(self, key, series): - valid_flvr_series = conf.flavor_series.valid_series + valid_flvr_series = cfg.CONF.fms.flavor_series if series not in valid_flvr_series: - raise ValueError("Series must be one of {}:".format(str(valid_flvr_series))) + raise ValueError( + "Series must be one of {}:".format(str(valid_flvr_series))) return series @@ -155,9 +158,10 @@ class Flavor(Base, FMSBaseModel): region_deleted_flag = True if not region_deleted_flag: - raise ErrorStatus(404, - "Failed to remove region {0} from flavor id {1}".format( - region_name, str(self.id))) + raise ErrorStatus( + 404, + "Failed to remove region {0} from flavor id {1}".format( + region_name, str(self.id))) except ErrorStatus as e: raise @@ -169,7 +173,8 @@ class Flavor(Base, FMSBaseModel): raise def add_tags(self, flavor_tags): - assert isinstance(flavor_tags, list) and all(isinstance(ft, FlavorTag) for ft in flavor_tags) + assert isinstance(flavor_tags, list) \ + and all(isinstance(ft, FlavorTag) for ft in flavor_tags) try: LOG.debug("add tags {0} to flavor {1}".format(str(flavor_tags), str(self))) @@ -182,16 +187,18 @@ class Flavor(Base, FMSBaseModel): raise def replace_tags(self, flavor_tags): - assert isinstance(flavor_tags, list) and all(isinstance(ft, FlavorTag) for ft in flavor_tags) + assert isinstance(flavor_tags, list) \ + and all(isinstance(ft, FlavorTag) for ft in flavor_tags) try: - LOG.debug("replace tags {0} for flavor {1}".format(str(flavor_tags), - str(self))) + LOG.debug("replace tags {0} for flavor {1}".format( + str(flavor_tags), str(self))) self.remove_all_tags() self.add_tags(flavor_tags) except Exception as exception: - LOG.log_exception("Failed to replace tags {0} to flavor {1}".format( - str(flavor_tags), str(self)), exception) + LOG.log_exception( + "Failed to replace tags {0} to flavor {1}".format( + str(flavor_tags), str(self)), exception) raise def remove_all_tags(self): @@ -199,7 +206,9 @@ class Flavor(Base, FMSBaseModel): LOG.debug("remove all tags from flavor {}".format(str(self))) self.flavor_tags = [] except Exception as exception: - LOG.log_exception("Failed to remove all tags from flavor {}".format(str(self)), exception) + LOG.log_exception( + "Failed to remove all tags from flavor {}".format( + str(self)), exception) raise def remove_tag(self, tag_name): @@ -215,9 +224,10 @@ class Flavor(Base, FMSBaseModel): deleted_flag = True if not deleted_flag: - raise ErrorStatus(404, - "Failed to remove tag {0} from flavor id {1}".format( - tag_name, str(self.id))) + raise ErrorStatus( + 404, + "Failed to remove tag {0} from flavor id {1}".format( + tag_name, str(self.id))) except ErrorStatus as e: raise @@ -254,9 +264,10 @@ class Flavor(Base, FMSBaseModel): deleted_flag = True if not deleted_flag: - raise ErrorStatus(404, - "tenant {0} does not exist for flavor id {1}".format( - tenant_id, str(self.id))) + raise ErrorStatus( + 404, + "tenant {0} does not exist for flavor id {1}".format( + tenant_id, str(self.id))) except Exception as exception: LOG.log_exception( @@ -293,8 +304,8 @@ class Flavor(Base, FMSBaseModel): deleted_flag = False assert isinstance(extra_spec_key_name, basestring) try: - LOG.debug("remove extra_spec {} from flavor {}".format(extra_spec_key_name, - str(self))) + LOG.debug("remove extra_spec {} from flavor {}".format( + extra_spec_key_name, str(self))) for extra_spec in reversed(self.flavor_extra_specs): if extra_spec.key_name == extra_spec_key_name: @@ -302,16 +313,18 @@ class Flavor(Base, FMSBaseModel): deleted_flag = True if not deleted_flag: - raise ErrorStatus(404, - "extra spec {0} does not exist for flavor id {1}".format( - extra_spec_key_name, str(self.id))) + raise ErrorStatus( + 404, + "extra spec {0} does not exist for flavor id {1}".format( + extra_spec_key_name, str(self.id))) except ErrorStatus as e: raise except Exception as exception: LOG.log_exception( - "Failed to remove extra_spec {0} from flavor {1}".format(extra_spec_key_name, str(self)), exception) + "Failed to remove extra_spec {0} from flavor {1}".format( + extra_spec_key_name, str(self)), exception) raise def validate(self, type): @@ -319,7 +332,8 @@ class Flavor(Base, FMSBaseModel): ''' if self.visibility == "public" and len(self.flavor_tenants) > 0: - raise ValueError("tenants should not be specified for a public flavor") + raise ValueError( + "tenants should not be specified for a public flavor") elif self.visibility == "private" and len(self.flavor_tenants) == 0: raise ValueError("Tenants must be specified for a private flavor") elif self.visibility not in ["private", "public"]: @@ -336,8 +350,8 @@ class Flavor(Base, FMSBaseModel): ''' -' FlavorExtraSpec is a DataObject contains all fields defined in FlavorExtraSpec -' defined as SqlAlchemy model map to a table +' FlavorExtraSpec is a DataObject contains all fields defined in +' FlavorExtraSpec defined as SqlAlchemy model map to a table ''' @@ -345,9 +359,13 @@ class FlavorExtraSpec(Base, FMSBaseModel): def __init__(self, key_name=None, - key_value=None): + key_value=None, + key_name_value=None): Base.__init__(self) + if key_name_value: + key_name, key_value = key_name_value.split(': ') + self.key_name = key_name self.key_value = key_value @@ -465,10 +483,10 @@ class FlavorRegion(Base, FMSBaseModel): region_type = Column(String) def __repr__(self): - text = "FlavorRegion(flavor_internal_id='{}', region_name='{}', region_type='{}')".format(self.flavor_internal_id, - self.region_name, - self.region_type - ) + text = "FlavorRegion(flavor_internal_id='{}', region_name='{}', " \ + "region_type='{}')".format(self.flavor_internal_id, + self.region_name, + self.region_type) return text def todict(self): diff --git a/orm/services/flavor_manager/fms_rest/data/wsme/models.py b/orm/services/flavor_manager/fms_rest/data/wsme/models.py index 07d4efa3..a8b5ddb4 100755 --- a/orm/services/flavor_manager/fms_rest/data/wsme/models.py +++ b/orm/services/flavor_manager/fms_rest/data/wsme/models.py @@ -1,4 +1,3 @@ -import ast import wsme from orm.common.orm_common.utils.cross_api_utils import (set_utils_conf, @@ -8,6 +7,7 @@ from orm.services.flavor_manager.fms_rest.data.sql_alchemy import db_models from orm.services.flavor_manager.fms_rest.data.wsme.model import Model from orm.services.flavor_manager.fms_rest.logic.error_base import ErrorStatus +from oslo_config import cfg from pecan import conf, request @@ -45,8 +45,9 @@ class ExtraSpecsWrapper(Model): def to_db_model(self): extra_spec = [] + autogen_es = cfg.CONF.flavor_series_metadata.autogen_extra_specs for key, value in self.os_extra_specs.iteritems(): - if Flavor.ignore_extra_specs_input(key.replace(":", "____")): + if key in autogen_es: continue extra_spec_rec = db_models.FlavorExtraSpec() extra_spec_rec.key_name = key @@ -65,7 +66,9 @@ class ExtraSpecsWrapper(Model): extra_specs = ExtraSpecsWrapper() setattr(extra_specs, extra_spec_method, {}) for extra_spec in os_extra_specs: - getattr(extra_specs, extra_spec_method, None)[extra_spec.key_name] = extra_spec.key_value + getattr(extra_specs, + extra_spec_method, + None)[extra_spec.key_name] = extra_spec.key_value return extra_specs @@ -228,48 +231,65 @@ class Flavor(Model): self.status = status def validate_model(self, context=None): + series_metadata = {} + valid_numa = [] + valid_vnf = [] - bundle = ['b1', 'b2'] - numa = ['n0'] - vlan = ['v1', 'v2', 'v3', 'v4', 'v5', 'v6'] + if self.series: + valid_flavor_series = cfg.CONF.fms.flavor_series + if self.series not in valid_flavor_series: + raise ErrorStatus(400, "Series possible values are {}".format( + valid_flavor_series)) + else: + raise ErrorStatus(400, "Series not specified.") + + if self.series in cfg.CONF['flavor_series_metadata']: + series_metadata = cfg.CONF['flavor_series_metadata'][self.series] + else: + raise ErrorStatus(400, "Cannot retrieve requested flavor" + " series metadata.") + + if 'valid_options_numa' in series_metadata: + valid_numa = [x for x in + series_metadata['valid_options_numa'].split(',')] + + if 'valid_options_vnf' in series_metadata: + valid_vnf = [x for x in + series_metadata['valid_options_vnf'].split(',')] # validate that input entries are valid try: # flavor option values must be either 'true' or 'false' (in quotes) - bools = ['true', 'false'] if self.options: option_values = self.options.values() invalid_opt_vals = [x for x in option_values if (x.lower() - not in bools)] + not in ['true', 'false'])] if invalid_opt_vals: raise ErrorStatus(400, "All flavor option values must have" " a value of 'true' or 'false'") - # validate series and set flavor limits - if self.series: - valid_flavor_series = conf.flavor_series.valid_series - if self.series not in valid_flavor_series: - raise ErrorStatus(400, "series possible values are {}".format( - valid_flavor_series)) + # validate series and set flavor vcpu and vram limits + requested_numa = [n for n in valid_numa if n in + self.options.keys() and + self.options[n].lower() == 'true'] - if self.series == 'p1': - if {'n0'}.issubset(self.options.keys()) and \ - ast.literal_eval(self.options.get('n0').lower().capitalize()): - vcpu_limit = int(conf.flavor_limits.p1_n0_vcpu_limit) - vram_limit = int(conf.flavor_limits.p1_n0_vram_limit) - else: - vcpu_limit = int(conf.flavor_limits.p1_nx_vcpu_limit) - vram_limit = int(conf.flavor_limits.p1_nx_vram_limit) + if requested_numa: + vcpu_limit = int(series_metadata['vcpu_limit']) + vram_limit = int(series_metadata['vram_limit']) + else: + vcpu_limit = int(series_metadata['alt_vcpu_limit']) + vram_limit = int(series_metadata['alt_vram_limit']) # determine other flavor limits - swap_file_limit = int(conf.flavor_limits.swap_file_limit) - ephemeral_limit = int(conf.flavor_limits.ephemeral_limit) + swap_file_limit = cfg.CONF.fms.flavor_swap_file_limit + ephemeral_limit = cfg.CONF.fms.flavor_ephemeral_limit isValid = validate_description(self.description) if not isValid: - raise ErrorStatus(400, "Flavor description does not allow special characters :" - " only dashes, commas, and period allowed.") + raise ErrorStatus(400, "Flavor description does not allow" + " special characters: only dashes," + " commas, and period allowed.") if not self.ram.isdigit(): raise ErrorStatus(400, "ram must be a number") if not self.vcpus.isdigit(): @@ -282,47 +302,53 @@ class Flavor(Model): raise ErrorStatus(400, "ephemeral must be a number") if int(self.ram) not in range(1024, vram_limit + 1, 1024): raise ErrorStatus(400, - "ram value is out of range. Expected range is 1024(1GB)-" - "%6d(%3dGB) and must be a multiple of 1024" % + "ram value is out of range. Expected range" + " is 1024(1GB)-%6d(%3dGB) and must be a" + " multiple of 1024" % (vram_limit, vram_limit / 1024)) if int(self.vcpus) not in range(1, vcpu_limit + 1): - raise ErrorStatus(400, "vcpus value is out of range. Expected range is 1-" - "%2d" % (vcpu_limit)) + raise ErrorStatus(400, "vcpus value is out of range. Expected" + "range is 1-%2d" % (vcpu_limit)) if int(self.disk) < 0: raise ErrorStatus(400, "disk cannot be less than zero") if not self.ephemeral: self.ephemeral = "0" - elif self.ephemeral and int(self.ephemeral) not in range(0, ephemeral_limit + 1): - raise ErrorStatus(400, "ephemeral value is out of range. Expected range is 0-" - "%5d(%2dTB)" % (ephemeral_limit, ephemeral_limit / 1000)) + elif (self.ephemeral and + int(self.ephemeral) not in range(0, ephemeral_limit + 1)): + raise ErrorStatus(400, + "ephemeral value is out of range. Expected" + " range is 0-%5d(%2dTB)" % + (ephemeral_limit, ephemeral_limit / 1000)) if int(self.swap) not in range(0, swap_file_limit + 1, 1024): raise ErrorStatus(400, - "swap value is out of range. Expected range is 0-" - "%6d(%3dGB) and must be a multiple of 1024" % + "swap value is out of range. Expected" + " range is 0-%6d(%3dGB) and must be a" + " multiple of 1024" % (swap_file_limit, swap_file_limit / 1024)) except ValueError: - raise ErrorStatus(400, "ram, vcpus, disk, ephemeral and swap must be integers") + raise ErrorStatus(400, "ram, vcpus, disk, ephemeral and swap must" + " be integers") for symbol, value in self.extra_specs.iteritems(): - if symbol == 'bundle' and value not in bundle: + if symbol == 'numa_override' and value not in valid_numa: raise ErrorStatus(400, - "Invalid value. bundle possible values: " + str(bundle)) - if symbol == 'numa_override' and value not in numa: + "Invalid value. numa_override possible" + " values: " + str(valid_numa)) + if symbol == 'vlan_category' and value not in valid_vnf: raise ErrorStatus(400, - "Invalid value. numa_override possible values: " + str(numa)) - if symbol == 'vlan_category' and value not in vlan: - raise ErrorStatus(400, - "Invalid value. vlan_category possible values: " + str(vlan)) + "Invalid value. vlan_category possible" + " values: " + str(valid_vnf)) - # region type can be group only in create flavor!! + # region type can be group only in create flavor!! if not context == "create": for region in self.regions: if region.type == "group": raise ErrorStatus(400, - "region type \'group\' is invalid in this " - "action, \'group\' can be only in create flavor action") + "region type \'group\' is invalid in" + " this action, \'group\' can be only" + " in create flavor action") def to_db_model(self): flavor = db_models.Flavor() @@ -331,9 +357,10 @@ class Flavor(Model): options = [] regions = [] tenants = [] + autogen_es = cfg.CONF.flavor_series_metadata.autogen_extra_specs for symbol, value in self.extra_specs.iteritems(): - if self.ignore_extra_specs_input(symbol.replace(":", "____")): + if symbol in autogen_es: continue es = db_models.FlavorExtraSpec() es.key_name = symbol @@ -384,15 +411,6 @@ class Flavor(Model): return flavor - @staticmethod - def ignore_extra_specs_input(symbol): - ignore_keys = conf.default_extra_spec_calculated_table.to_dict() - if symbol in ignore_keys: - return True - if len(symbol) == 36 and symbol[0:35] in ignore_keys and symbol[35].isdigit() and 1 <= int(symbol[35]) <= 8: - return True - return False - @staticmethod def from_db_model(sql_flavor): flavor = Flavor() @@ -429,62 +447,71 @@ class Flavor(Model): return flavor - @staticmethod - def ignore_extra_specs_input(symbol): - ignore_keys = conf.default_extra_spec_calculated_table.to_dict() - if symbol in ignore_keys: - return True - if len(symbol) == 36 and symbol[0:35] in ignore_keys and symbol[35].isdigit() and 1 <= int(symbol[35]) <= 8: - return True - return False - def get_extra_spec_needed(self): extra_spec_needed = [] - items = conf.extra_spec_needed_table.to_dict() - for symbol, value in items[self.series].iteritems(): - es = db_models.FlavorExtraSpec() - es.key_name = symbol.replace("____", ":") - es.key_value = value + requested_options = [] + mixed_options = {} + series_metadata = cfg.CONF['flavor_series_metadata'][self.series] - extra_spec_needed.append(es) + # Retreive default extra specs and mixed options for series + for f_key, f_val in series_metadata.items(): + if f_key.startswith("es_default_"): + es = db_models.FlavorExtraSpec(key_name_value=f_val) + extra_spec_needed.append(es) + if f_key.startswith("es_mixed_"): + mixed_es_trimmed = f_key.replace('es_mixed_', '') + mixed_options[mixed_es_trimmed] = f_val - options_items = self.options - # check some keys if they exist in option add values to extra specs - if self.series in 'p1': - n0_in = False - for symbol, value in options_items.iteritems(): - es = db_models.FlavorExtraSpec() - es.key_name = "aggregate_instance_extra_specs:%s" % symbol - es.key_value = "true" - # format numa node extra spec as appropriate - if symbol == "n0" and options_items[symbol].lower() == "true": - n0_in = True - es.key_value = 2 - es.key_name = "hw:numa_nodes" - extra_spec_needed.append(es) + # Evaluate numa options + if 'valid_options_numa' in series_metadata: + valid_numa = [x for x in + series_metadata['valid_options_numa'].split(',')] - # add the default extra specs - es = db_models.FlavorExtraSpec() - es.key_name = "hw:cpu_policy" - es.key_value = "dedicated" - extra_spec_needed.append(es) + option_numa = [n for n in valid_numa if n in + self.options.keys() and + self.options[n].lower() == 'true'] - if not n0_in: - es = db_models.FlavorExtraSpec() - es.key_value = 1 - es.key_name = "hw:numa_nodes" + if not option_numa: + es = db_models.FlavorExtraSpec( + key_name_value=series_metadata['es_alt_numa_nodes']) + extra_spec_needed.append(es) + else: + es = db_models.FlavorExtraSpec( + key_name_value=series_metadata['es_numa_nodes']) + extra_spec_needed.append(es) + requested_options.extend(option_numa) + + # Evaluate pci options + if 'valid_options_pci' in series_metadata: + valid_pci = [x for x in + series_metadata['valid_options_pci'].split(',')] + + option_pci = [n for n in valid_pci if n in + self.options.keys() and + self.options[n].lower() == 'true'] + + if option_pci: + requested_options.extend(option_pci) + + # Evalulate mixed options + assorted_opts = [] + for mixed_key, mixed_value in mixed_options.items(): + assorted_opts = [z for z in mixed_key.split('_')] + + mixed_present = True + for opt in assorted_opts: + mixed_present &= True if opt in requested_options else False + + if mixed_present: + es = db_models.FlavorExtraSpec(key_name_value=mixed_value) extra_spec_needed.append(es) - if self.series in ['p1'] and {'i2'}.issubset(options_items.keys()): - es = db_models.FlavorExtraSpec() - es.key_name = "hw:pci_numa_affinity_policy" - es.key_value = "dedicated" - extra_spec_needed.append(es) - - # convert the key_value to a string to avoid/fix pecan json rendering error in update extra_specs + # convert the key_value to a string to avoid/fix pecan json + # rendering error in update extra_specs i = 0 while i < len(extra_spec_needed): - extra_spec_needed[i].key_value = str(extra_spec_needed[i].key_value) + extra_spec_needed[i].key_value = str( + extra_spec_needed[i].key_value) i += 1 return extra_spec_needed @@ -498,18 +525,19 @@ class Flavor(Model): def handle_region_groups(self): regions_to_add = [] - for region in self.regions[:]: # get copy of it to be able to delete from the origin + # get copy of it to be able to delete from the origin + for region in self.regions[:]: if region.type == "group": group_regions = self.get_regions_for_group(region.name) if group_regions is None: raise ValueError("Group {} not found".format(region.name)) - # if len(group_regions) == 0: - # raise ValueError("Group {} is empty, no region was assigned to it".format(region.name)) for group_region in group_regions: - regions_to_add.append(Region(name=group_region, type='single')) + regions_to_add.append(Region(name=group_region, + type='single')) self.regions.remove(region) - self.regions.extend(set(regions_to_add)) # remove duplicates if exist + # remove duplicates if exist + self.regions.extend(set(regions_to_add)) def get_regions_for_group(self, group_name): set_utils_conf(conf) diff --git a/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py b/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py index 5a28cc3c..1d4ca2d2 100755 --- a/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py +++ b/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py @@ -1,13 +1,16 @@ -from orm.services.flavor_manager.fms_rest.data.sql_alchemy.db_models import FlavorRegion, FlavorTenant -from orm.services.flavor_manager.fms_rest.data.wsme.models import (ExtraSpecsWrapper, Flavor, - FlavorListFullResponse, FlavorWrapper, - Region, RegionWrapper, TagsWrapper, - TenantWrapper) +from orm.services.flavor_manager.fms_rest.data.sql_alchemy.db_models import ( + FlavorRegion, FlavorTenant) +from orm.services.flavor_manager.fms_rest.data.wsme.models import ( + ExtraSpecsWrapper, Flavor, + FlavorListFullResponse, FlavorWrapper, + Region, RegionWrapper, TagsWrapper, + TenantWrapper) from orm.services.flavor_manager.fms_rest.logger import get_logger -from orm.services.flavor_manager.fms_rest.logic.error_base import ConflictError, ErrorStatus, NotFoundError +from orm.services.flavor_manager.fms_rest.logic.error_base import ( + ConflictError, ErrorStatus, NotFoundError) from orm.common.orm_common.injector import injector -from pecan import conf +from oslo_config import cfg LOG = get_logger(__name__) @@ -35,9 +38,10 @@ def create_flavor(flavor, flavor_uuid, transaction_id): datamanager.begin_transaction() flavor_rec.insert(sql_flavor) - datamanager.flush() # i want to get any exception created by this insert + datamanager.flush() # Get any exception created by this insert existing_region_names = [] - send_to_rds_if_needed(sql_flavor, existing_region_names, "post", transaction_id) + send_to_rds_if_needed( + sql_flavor, existing_region_names, "post", transaction_id) datamanager.commit() @@ -45,25 +49,33 @@ def create_flavor(flavor, flavor_uuid, transaction_id): return ret_flavor except Exception as exp: - LOG.log_exception("FlavorLogic - Failed to CreateFlavor", str(exp.args)) + LOG.log_exception("FlavorLogic - Failed to CreateFlavor", + str(exp.args)) datamanager.rollback() if "Duplicate entry" in exp.args: - raise ConflictError(409, "Flavor {} already exists".format(flavor.flavor.name)) + raise ConflictError( + 409, + "Flavor {} already exists".format(flavor.flavor.name)) raise finally: datamanager.close() @di.dependsOn('rds_proxy') -def send_to_rds_if_needed(sql_flavor, existing_region_names, http_action, transaction_id): +def send_to_rds_if_needed(sql_flavor, + existing_region_names, + http_action, + transaction_id): rds_proxy = di.resolver.unpack(send_to_rds_if_needed) - if (sql_flavor.flavor_regions and len(sql_flavor.flavor_regions) > 0) or len(existing_region_names) > 0: + if (sql_flavor.flavor_regions and len(sql_flavor.flavor_regions) > 0) \ + or len(existing_region_names) > 0: flavor_dict = sql_flavor.todict() update_region_actions(flavor_dict, existing_region_names, http_action) LOG.debug("Flavor is valid to send to RDS - sending to RDS Proxy ") rds_proxy.send_flavor(flavor_dict, transaction_id, http_action) else: - LOG.debug("Flavor with no regions - wasn't sent to RDS Proxy " + str(sql_flavor)) + LOG.debug("Flavor with no regions - wasn't sent to RDS Proxy " + + str(sql_flavor)) @di.dependsOn('data_manager') @@ -91,9 +103,11 @@ def update_flavor(flavor, flavor_uuid, transaction_id): # pragma: no cover del(db_flavor) flavor_rec.insert(sql_flavor) - datamanager.flush() # i want to get any exception created by this insert method + # get any exception created by this insert method + datamanager.flush() - send_to_rds_if_needed(sql_flavor, existing_region_names, "put", transaction_id) + send_to_rds_if_needed( + sql_flavor, existing_region_names, "put", transaction_id) datamanager.commit() @@ -120,7 +134,8 @@ def delete_flavor_by_uuid(flavor_uuid): # , transaction_id): sql_flavor = flavor_rec.get_flavor_by_id(flavor_uuid) if sql_flavor is None: - raise NotFoundError(message="Flavor '{}' not found".format(flavor_uuid)) + raise NotFoundError( + message="Flavor '{}' not found".format(flavor_uuid)) existing_region_names = sql_flavor.get_existing_region_names() if len(existing_region_names) > 0: @@ -139,7 +154,7 @@ def delete_flavor_by_uuid(flavor_uuid): # , transaction_id): status_resp = resp.json() if 'status' in status_resp.keys(): LOG.debug( - 'RDS returned status: {}'.format(status_resp['status'])) + 'RDS returned status:{}'.format(status_resp['status'])) status = status_resp['status'] else: # Invalid response from RDS @@ -158,7 +173,8 @@ def delete_flavor_by_uuid(flavor_uuid): # , transaction_id): if status == invalid_status: LOG.error('Invalid flavor status received from RDS') - raise ErrorStatus(500, "Invalid flavor status received from RDS") + raise ErrorStatus(500, + "Invalid flavor status received from RDS") elif status != expected_status: msg = "The flavor has not been deleted " \ "successfully from all of its regions " \ @@ -170,7 +186,8 @@ def delete_flavor_by_uuid(flavor_uuid): # , transaction_id): # OK to delete flavor_rec.delete_by_uuid(flavor_uuid) - datamanager.flush() # i want to get any exception created by this delete + # get any exception created by this delete + datamanager.flush() datamanager.commit() except Exception as exp: LOG.log_exception("FlavorLogic - Failed to delete flavor", exp) @@ -189,7 +206,8 @@ def add_regions(flavor_uuid, regions, transaction_id): flavor_rec = datamanager.get_record('flavor') sql_flavor = flavor_rec.get_flavor_by_id(flavor_uuid) if not sql_flavor: - raise ErrorStatus(404, 'flavor id {0} not found'.format(flavor_uuid)) + raise ErrorStatus(404, + 'flavor id {0} not found'.format(flavor_uuid)) existing_region_names = sql_flavor.get_existing_region_names() @@ -197,13 +215,18 @@ def add_regions(flavor_uuid, regions, transaction_id): if region.name == '' or region.name.isspace(): raise ErrorStatus(400, 'Cannot add region with an empty name') if region.type == "group": - raise ErrorStatus(400, 'Adding \'group\' type region is supported only when creating a flavor') - db_region = FlavorRegion(region_name=region.name, region_type='single') + raise ErrorStatus(400, + "Adding \'group\' type region is supported" + " only when creating a flavor") + db_region = FlavorRegion(region_name=region.name, + region_type='single') sql_flavor.add_region(db_region) - datamanager.flush() # i want to get any exception created by previous actions against the database + # get any exception created by previous actions against the database + datamanager.flush() - send_to_rds_if_needed(sql_flavor, existing_region_names, "put", transaction_id) + send_to_rds_if_needed( + sql_flavor, existing_region_names, "put", transaction_id) datamanager.commit() @@ -219,7 +242,8 @@ def add_regions(flavor_uuid, regions, transaction_id): LOG.log_exception("FlavorLogic - Failed to add regions", exp) datamanager.rollback() if "conflicts with persistent instance" in str(exp.args): - raise ConflictError(409, "One or more regions already exists in Flavor") + raise ConflictError(409, + "One or more regions already exists in Flavor") raise exp finally: datamanager.close() @@ -237,12 +261,14 @@ def delete_region(flavor_uuid, region_name, transaction_id, on_success_by_rds, if not sql_flavor and on_success_by_rds: return if not sql_flavor: - raise ErrorStatus(404, 'flavor id {0} not found'.format(flavor_uuid)) + raise ErrorStatus(404, + 'flavor id {0} not found'.format(flavor_uuid)) existing_region_names = sql_flavor.get_existing_region_names() sql_flavor.remove_region(region_name) - datamanager.flush() # Get any exception created by previous actions against the database + # get any exception created by previous actions against the database + datamanager.flush() if on_success_by_rds: datamanager.commit() else: @@ -276,7 +302,8 @@ def add_tenants(flavor_uuid, tenants, transaction_id): flavor_rec = datamanager.get_record('flavor') sql_flavor = flavor_rec.get_flavor_by_id(flavor_uuid) if not sql_flavor: - raise ErrorStatus(404, 'Flavor id {0} not found'.format(flavor_uuid)) + raise ErrorStatus(404, + 'Flavor id {0} not found'.format(flavor_uuid)) if sql_flavor.visibility == "public": raise ErrorStatus(405, 'Cannot add tenant to a public flavor') @@ -285,13 +312,15 @@ def add_tenants(flavor_uuid, tenants, transaction_id): for tenant in tenants.tenants: if not isinstance(tenant, basestring): - raise ValueError("tenant type must be a string type, got {} type".format(type(tenant))) + raise ValueError("tenant type must be a string type," + " got {} type".format(type(tenant))) db_tenant = FlavorTenant(tenant_id=tenant) sql_flavor.add_tenant(db_tenant) - - datamanager.flush() # i want to get any exception created by previous actions against the database - send_to_rds_if_needed(sql_flavor, existing_region_names, "put", transaction_id) + # get any exception created by previous actions against the database + datamanager.flush() + send_to_rds_if_needed( + sql_flavor, existing_region_names, "put", transaction_id) datamanager.commit() flavor = get_flavor_by_uuid(flavor_uuid) @@ -321,20 +350,24 @@ def delete_tenant(flavor_uuid, tenant_id, transaction_id): flavor_rec = datamanager.get_record('flavor') sql_flavor = flavor_rec.get_flavor_by_id(flavor_uuid) if not sql_flavor: - raise ErrorStatus(404, 'flavor id {0} not found'.format(flavor_uuid)) + raise ErrorStatus(404, + 'flavor id {0} not found'.format(flavor_uuid)) # if trying to delete the only one tenant then return value error if sql_flavor.visibility == "public": - raise ValueError("{} is a public flavor, delete tenant action is not relevant".format(flavor_uuid)) + raise ValueError("{} is a public flavor, delete tenant" + " action is not relevant".format(flavor_uuid)) - if len(sql_flavor.flavor_tenants) == 1 and sql_flavor.flavor_tenants[0].tenant_id == tenant_id: + if len(sql_flavor.flavor_tenants) == 1 \ + and sql_flavor.flavor_tenants[0].tenant_id == tenant_id: raise ValueError( 'Private flavor must have at least one tenant') existing_region_names = sql_flavor.get_existing_region_names() sql_flavor.remove_tenant(tenant_id) - - datamanager.flush() # i want to get any exception created by previous actions against the database - send_to_rds_if_needed(sql_flavor, existing_region_names, "put", transaction_id) + # get any exception created by previous actions against the database + datamanager.flush() + send_to_rds_if_needed( + sql_flavor, existing_region_names, "put", transaction_id) datamanager.commit() except NotFoundError as exp: datamanager.rollback() @@ -407,7 +440,6 @@ def delete_extra_specs(flavor_id, transaction_id, extra_spec=None): flavor_rec = datamanager.get_record("flavor") sql_flavor = flavor_rec.get_flavor_by_id(flavor_id) - if not sql_flavor: raise NotFoundError(404, 'flavor id {0} not found'.format( flavor_id)) @@ -426,8 +458,8 @@ def delete_extra_specs(flavor_id, transaction_id, extra_spec=None): else: sql_flavor.delete_all_extra_specs() sql_flavor.add_extra_specs(default_extra_specs) - - datamanager.flush() # i want to get any exception created by previous actions against the database + # get any exception created by previous actions against the database + datamanager.flush() send_to_rds_if_needed(sql_flavor, existing_region_names, "put", transaction_id) datamanager.commit() @@ -443,7 +475,8 @@ def delete_extra_specs(flavor_id, transaction_id, extra_spec=None): LOG.log_exception("FlavorLogic - extra specs not found", exp) raise else: - LOG.log_exception("FlavorLogic - failed to delete extra specs", exp) + LOG.log_exception("FlavorLogic - failed to delete extra specs", + exp) raise except Exception as exp: @@ -488,7 +521,8 @@ def delete_tags(flavor_id, tag, transaction_id): sql_flavor = flavor_rec.get_flavor_by_id(flavor_id) if not sql_flavor: - raise NotFoundError(404, 'flavor id {0} not found'.format(flavor_id)) + raise NotFoundError(404, + 'flavor id {0} not found'.format(flavor_id)) if tag: sql_flavor.remove_tag(tag) @@ -577,7 +611,8 @@ def add_extra_specs(flavor_id, extra_specs, transaction_id): extra_specs_model = extra_specs.to_db_model() sql_flavor.add_extra_specs(extra_specs_model) - datamanager.flush() # i want to get any exception created by previous actions against the database + # get any exception created by previous actions against the database + datamanager.flush() send_to_rds_if_needed(sql_flavor, existing_region_names, "put", transaction_id) datamanager.commit() @@ -592,7 +627,10 @@ def add_extra_specs(flavor_id, extra_specs, transaction_id): except Exception as exp: datamanager.rollback() if "conflicts with persistent instance" in str(exp.args): - raise ConflictError(409, "one or all extra specs {} already exists".format(extra_specs.os_extra_specs)) + raise ConflictError( + 409, + "one or all extra specs {} already" + " exists".format(extra_specs.os_extra_specs)) LOG.log_exception("FlavorLogic - fail to add extra spec", exp) raise finally: @@ -704,8 +742,8 @@ def add_tags(flavor_id, tags, transaction_id): except Exception as exp: datamanager.rollback() if "conflicts with persistent instance" in str(exp.args): - raise ConflictError(409, "one or all tags {} already exists".format( - tags.tags)) + raise ConflictError( + 409, "one or all tags {} already exists".format(tags.tags)) LOG.log_exception("FlavorLogic - fail to add tags", exp) raise finally: @@ -724,10 +762,13 @@ def get_flavor_by_uuid_or_name(flavor_uuid_or_name): flavor_record = datamanager.get_record('flavor') - sql_flavor = flavor_record.get_flavor_by_id_or_name(flavor_uuid_or_name) + sql_flavor = flavor_record.get_flavor_by_id_or_name( + flavor_uuid_or_name) if not sql_flavor: - raise ErrorStatus(404, 'flavor with id or name {0} not found'.format(flavor_uuid_or_name)) + raise ErrorStatus( + 404, + 'flavor id or name {0} not found'.format(flavor_uuid_or_name)) flavor_wrapper = FlavorWrapper.from_db_model(sql_flavor) @@ -744,9 +785,10 @@ def get_flavor_by_uuid_or_name(flavor_uuid_or_name): @di.dependsOn('rds_proxy') def update_region_statuses(flavor, sql_flavor): rds_proxy = di.resolver.unpack(update_region_statuses) - # remove the regions comes from database and return the regions which return from rds, - # because there might be group region there (in the db) and in the response from the - # rds we will have a list of all regions belong to this group + # remove the regions comes from database and return the regions which + # return from rds, because there might be group region there (in the db) + # and in the response from the rds we will have a list of all regions + # belong to this group flavor.regions[:] = [] resp = rds_proxy.get_status(sql_flavor.id) if resp.status_code == 200: @@ -759,7 +801,8 @@ def update_region_statuses(flavor, sql_flavor): # get region status if region in flavor_regions_list if flavor_regions_list and 'regions' in rds_status_resp.keys(): for rds_region_status in rds_status_resp['regions']: - # check that the region read from RDS is in the flavor_regions_list + # check that the region read from RDS is in the + # flavor_regions_list if rds_region_status['region'] in flavor_regions_list: flavor.regions.append( Region(name=rds_region_status['region'], type="single", @@ -767,7 +810,8 @@ def update_region_statuses(flavor, sql_flavor): error_message=rds_region_status['error_msg'])) if 'status' in rds_status_resp.keys(): - # if flavor not assigned to region set flavor.status to 'no regions' + # if flavor not assigned to region set flavor.status to + # 'no regions' if flavor_regions_list: flavor.status = rds_status_resp['status'] else: @@ -790,37 +834,47 @@ def get_flavor_list_by_params(visibility, region, tenant, series, vm_type, try: flavor_record = datamanager.get_record('flavor') - sql_flavors = flavor_record.get_flavors_by_criteria(visibility=visibility, - region=region, - tenant=tenant, - series=series, - vm_type=vm_type, - vnf_name=vnf_name, - starts_with=starts_with, - contains=contains, - alias=alias) + sql_flavors = flavor_record.get_flavors_by_criteria( + visibility=visibility, + region=region, + tenant=tenant, + series=series, + vm_type=vm_type, + vnf_name=vnf_name, + starts_with=starts_with, + contains=contains, + alias=alias) response = FlavorListFullResponse() if sql_flavors: uuids = ','.join(str("\'" + sql_flavor.id + "\'") - for sql_flavor in sql_flavors if sql_flavor and sql_flavor.id) - resource_status_dict = flavor_record.get_flavors_status_by_uuids(uuids) + for sql_flavor in sql_flavors + if sql_flavor and sql_flavor.id) + + resource_status_dict = flavor_record.get_flavors_status_by_uuids( + uuids) for sql_flavor in sql_flavors: flavor = Flavor.from_db_model(sql_flavor) if sql_flavor.id: - # rds_region_list contains tuples - each containing the region associated - # with the flavor along with the region status + # rds_region_list contains tuples - each containing the + # region associated with the flavor along with the region + # status rds_region_list = resource_status_dict.get(sql_flavor.id) - # determine flavor overall status by checking its region statuses: + # determine flavor overall status by checking its region + # statuses: if rds_region_list and flavor.regions: - # set image.status to 'error' if any of the regions has an 'Error' status' - # else, if any region status shows 'Submitted' then set image status to 'Pending' + # set image.status to 'error' if any of the regions has + # an 'Error' status' else, if any region status shows + # 'Submitted' then set image status to 'Pending' # otherwise image status = 'Success' - error_status = [item for item in rds_region_list if item[1] == 'Error'] - submitted_status = [item for item in rds_region_list if item[1] == 'Submitted'] - success_status = [item for item in rds_region_list if item[1] == 'Success'] + error_status = [item for item in rds_region_list + if item[1] == 'Error'] + submitted_status = [item for item in rds_region_list + if item[1] == 'Submitted'] + success_status = [item for item in rds_region_list + if item[1] == 'Success'] if len(error_status) > 0: flavor.status = 'Error' @@ -829,7 +883,8 @@ def get_flavor_list_by_params(visibility, region, tenant, series, vm_type, elif len(success_status) > 0: flavor.status = 'Success' - # use rds_region_list to format the regions' statuses in flavor record + # use rds_region_list to format the regions' statuses + # in flavor record for rgn in flavor.regions: for rds_row_items in rds_region_list: if rgn.name == rds_row_items[0]: @@ -851,43 +906,31 @@ def get_flavor_list_by_params(visibility, region, tenant, series, vm_type, def calculate_name(flavor): """ calculate_name function returns the ranger flavor_name: - Ranger flavor name is made up of the following components, each separated by a DOT separator: + Ranger flavor name is made up of the following components, each separated + by a DOT separator: - flavor series - flavor attributes (cores, ram, disk, swap file, ephemeral disks) - flavor options (OPTIONAL) Following is a sample flavor name: - name = p1.c1r1d1s4e5.i2n0 + name = xx.c1r1d1s4e5.i2n0 - where p1 : flavor series name of 'p1' + where xx : flavor series name c1 : number of cores (or cpu); sample shows vcpu = 1 r1 : RAM value divided by units of 1024 MB; sample ram = 1024 (in MB) d3 : disk value measured in units of GB; sample shows disk = 3 - s4 : (optional) swap disk value divided by units of 1024 MB; sample swap value = 4096 (in MB) - e5 : (optional) ephemeral disk measured in units of GB - sample shows ephemeral = 5 (in GB) + s4 : (optional) swap disk value divided by units of 1024 MB; + sample swap value = 4096 (in MB) + e5 : (optional) ephemeral disk measured in units of GB - + sample shows ephemeral = 5 (in GB) - Anything after the second dot separator are flavor options 'i2' and 'n0' - flavor options are OPTIONAL + Anything after the second dot separator are flavor options 'i2' + and 'n0' - flavor options are OPTIONAL """ - valid_p1_opts = conf.flavor_options.valid_p1_opt_values[:] - name = "{0}.c{1}r{2}d{3}".format(flavor.flavor.series, flavor.flavor.vcpus, int(flavor.flavor.ram) / 1024, flavor.flavor.disk) - series = name[:2] - - # validate if a flavor is assigned with incompatible or invalid flavor options - n0_in = True - - if series in 'p1': - if ({'n0'}.issubset(flavor.flavor.options.keys()) and - flavor.flavor.options['n0'].lower() == 'false') or \ - not ({'n0'}.issubset(flavor.flavor.options.keys())): - n0_in = False - - if {'i2'}.issubset(flavor.flavor.options.keys()) and not n0_in: - raise ConflictError(409, "Flavor option i2 must be used with " - "flavor option 'n0' set to 'true'") # add swap disk info to flavor name IF provided swap = getattr(flavor.flavor, 'swap', 0) @@ -900,13 +943,22 @@ def calculate_name(flavor): name += '{}{}'.format('e', ephemeral) # add the valid option keys to the flavor name - if len(flavor.flavor.options) > 0: - for key in sorted(flavor.flavor.options): - # only include valid option parameters in flavor name - if series == 'p1' and key in valid_p1_opts and n0_in: - if name.count('.') < 2: - name += '.' - name += key + series_metadata = cfg.CONF['flavor_series_metadata'][flavor.flavor.series] + + valid_options = [series_metadata[x] for x in + series_metadata if x.startswith("valid_options_")] + + options = [n for n in valid_options if n in + flavor.flavor.options.keys() and + flavor.flavor.options[n].lower() == 'true'] + + if 'i2' in options and 'n0' not in options: + options.remove('i2') + + if options: + name += '.' + for key in sorted(options): + name += key return name @@ -925,10 +977,12 @@ def update_region_actions(flavor_dict, existing_region_names, action="put"): # add deleted regions for exist_region_name in existing_region_names: - if region_name_exist_in_regions(exist_region_name, flavor_dict["regions"]): + if region_name_exist_in_regions(exist_region_name, + flavor_dict["regions"]): continue else: - flavor_dict["regions"].append({"name": exist_region_name, "action": "delete"}) + flavor_dict["regions"].append({"name": exist_region_name, + "action": "delete"}) def region_name_exist_in_regions(region_name, regions): diff --git a/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql b/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql index 61eb698e..fe0dab1c 100755 --- a/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql +++ b/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql @@ -13,7 +13,7 @@ create table if not exists flavor name varchar(250) not null, alias varchar(64) null, description varchar(100) not null, - series enum('p1') not null, + series char(2) not null, ram integer not null, vcpus integer not null, disk integer not null, diff --git a/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_update_db.sql b/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_update_db.sql new file mode 100644 index 00000000..900eb4a3 --- /dev/null +++ b/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_update_db.sql @@ -0,0 +1,5 @@ +SET sql_notes=0; + +use orm; + +alter table flavor modify series char(2) NOT NULL; \ No newline at end of file diff --git a/orm/tests/config.py b/orm/tests/config.py index 3b2a4da9..7a68de55 100644 --- a/orm/tests/config.py +++ b/orm/tests/config.py @@ -25,8 +25,16 @@ app = { logging = { 'root': {'level': 'INFO', 'handlers': ['console']}, 'loggers': { - 'fms_rest': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, - 'pecan': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, + 'fms_rest': { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False + }, + 'pecan': { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False + }, 'py.warnings': {'handlers': ['console']}, '__force_dict__': True }, @@ -59,50 +67,11 @@ database = { } -# this table is for calculating default extra specs needed -extra_spec_needed_table = { - "ns": { - "aggregate_instance_extra_specs____ns": "true", - "hw____mem_page_size": "large" - }, - "nd": { - "aggregate_instance_extra_specs____nd": "true", - "hw____mem_page_size": "large" - }, - "nv": { - "aggregate_instance_extra_specs____nv": "true", - "hw____mem_page_size": "large" - }, - "gv": { - "aggregate_instance_extra_specs____gv": "true", - "aggregate_instance_extra_specs____c2": "true", - "hw____numa_nodes": "2" - }, - "ss": { - "aggregate_instance_extra_specs____ss": "true" - } -} - -# any key will be added to extra_spec_needed_table need to be added here -default_extra_spec_calculated_table = { - "aggregate_instance_extra_specs____ns": "", - "aggregate_instance_extra_specs____nd": "", - "aggregate_instance_extra_specs____nv": "", - "aggregate_instance_extra_specs____gv": "", - "aggregate_instance_extra_specs____c2": "", - "aggregate_instance_extra_specs____ss": "", - "aggregate_instance_extra_specs____c2": "", - "aggregate_instance_extra_specs____c4": "", - "aggregate_instance_extra_specs____v": "", - "hw____mem_page_size": "", - "hw____cpu_policy": "", - "hw____numa_nodes": "" -} - -database['connection_string'] = 'mysql://{0}:{1}@{2}:3306/{3}'.format(database['username'], - database['password'], - database['host'], - database['db_name']) +database['connection_string'] = 'mysql://{0}:{1}@{2}:3306/{3}'.format( + database['username'], + database['password'], + database['host'], + database['db_name']) application_root = 'http://localhost:{0}'.format(server['port']) diff --git a/orm/tests/unit/fms/config.py b/orm/tests/unit/fms/config.py index fa4588d3..8418b393 100755 --- a/orm/tests/unit/fms/config.py +++ b/orm/tests/unit/fms/config.py @@ -59,37 +59,6 @@ database = { } -flavor_series = { - 'valid_series': [ - 'p1' - ] -} - -# valid_flavor_options -flavor_options = { - - 'valid_p1_numa_value': 'n0', - 'valid_p1_opt_values': [ - 'n0', 'i2' - ] -} - -# this table is for calculating default extra specs needed -extra_spec_needed_table = { - "p1": { - "aggregate_instance_extra_specs____p1": "true", - "hw____mem_page_size": "large" - } -} - -# any key will be added to extra_spec_needed_table need to be added here -default_extra_spec_calculated_table = { - "aggregate_instance_extra_specs____p1": "", - "hw____mem_page_size": "", - "hw____cpu_policy": "", - "hw____numa_nodes": "" -} - database['connection_string'] = 'mysql://{0}:{1}@{2}:3306/{3}'.format(database['username'], database['password'], database['host'], diff --git a/orm/tests/unit/fms/test_flavor_logic.py b/orm/tests/unit/fms/test_flavor_logic.py index 7e76fc38..3e38e302 100755 --- a/orm/tests/unit/fms/test_flavor_logic.py +++ b/orm/tests/unit/fms/test_flavor_logic.py @@ -9,6 +9,8 @@ from orm.tests.unit.fms import FunctionalTest from sqlalchemy.orm import exc from mock import MagicMock, patch +from oslo_config import cfg + class OES(): @@ -62,7 +64,7 @@ class TestFlavorLogic(FunctionalTest): self.assertEqual(res_flavor.flavor.profile, 'N1') self.assertEqual(res_flavor.flavor.ram, '1024') self.assertEqual(res_flavor.flavor.vcpus, '1') - self.assertEqual(res_flavor.flavor.series, 'p1') + self.assertEqual(res_flavor.flavor.series, cfg.CONF.fms.flavor_series[0]) self.assertEqual(res_flavor.flavor.id, 'g') flavor = get_flavor_mock() @@ -75,8 +77,8 @@ class TestFlavorLogic(FunctionalTest): 'transaction') self.assertEqual(res_flavor.flavor.profile, 'N1') self.assertEqual(res_flavor.flavor.ram, '1024') - self.assertEqual(res_flavor .flavor.vcpus, '1') - self.assertEqual(res_flavor.flavor.series, 'p1') + self.assertEqual(res_flavor.flavor.vcpus, '1') + self.assertEqual(res_flavor.flavor.series, cfg.CONF.fms.flavor_series[0]) self.assertEqual(res_flavor.flavor.id, 'g') # @@ -719,7 +721,10 @@ def get_rds_proxy_mock(): def get_flavor_mock(): flavor_mock = FlavorWrapper() - flavor_mock.flavor = Flavor(ram='1024', vcpus='1', series='p1', id='g') + flavor_mock.flavor = Flavor(ram='1024', + vcpus='1', + series=cfg.CONF.fms.flavor_series[0], + id='g') flavor_mock.flavor.profile = 'N1' return flavor_mock diff --git a/orm/tests/unit/fms/test_wsme_models.py b/orm/tests/unit/fms/test_wsme_models.py index df8049cc..b3dd10ec 100755 --- a/orm/tests/unit/fms/test_wsme_models.py +++ b/orm/tests/unit/fms/test_wsme_models.py @@ -2,6 +2,7 @@ from orm.services.flavor_manager.fms_rest.data.sql_alchemy import db_models from orm.services.flavor_manager.fms_rest.data.wsme import models from orm.tests.unit.fms import FunctionalTest +from oslo_config import cfg class TestWsmeModels(FunctionalTest): @@ -11,8 +12,9 @@ class TestWsmeModels(FunctionalTest): sql_flavor.description = 'desc' sql_flavor.disk = 1 sql_flavor.ephemeral = 1 - sql_flavor.flavor_extra_specs = [db_models.FlavorExtraSpec('key1', 'val1'), - db_models.FlavorExtraSpec('key2', 'val2')] + sql_flavor.flavor_extra_specs = [db_models.FlavorExtraSpec( + 'key1', 'val1'), + db_models.FlavorExtraSpec('key2', 'val2')] sql_flavor.flavor_tag = [db_models.FlavorExtraSpec('key1', 'val1'), db_models.FlavorExtraSpec('key2', 'val2')] sql_flavor.flavor_options = [db_models.FlavorExtraSpec('key1', 'val1'), @@ -26,7 +28,7 @@ class TestWsmeModels(FunctionalTest): sql_flavor.ram = 1 sql_flavor.visibility = 'visibility' sql_flavor.vcpus = 1 - sql_flavor.series = "p1" + sql_flavor.series = cfg.CONF.fms.flavor_series[0] sql_flavor.swap = 1 sql_flavor.disk = 1 sql_flavor.name = 'name' @@ -58,14 +60,15 @@ class TestWsmeModels(FunctionalTest): flavor_wrapper.flavor.swap = '1' flavor_wrapper.flavor.disk = '1' flavor_wrapper.flavor.name = 'name' - flavor_wrapper.flavor.series = 'p1' + flavor_wrapper.flavor.series = cfg.CONF.fms.flavor_series[0] sql_flavor = flavor_wrapper.to_db_model() self.assertEqual(len(sql_flavor.flavor_regions), 2) self.assertEqual(len(sql_flavor.flavor_tenants), 2) - spec = next(s for s in sql_flavor.flavor_extra_specs if s.key_name == 'key1') + spec = next( + s for s in sql_flavor.flavor_extra_specs if s.key_name == 'key1') self.assertEqual(spec.key_value, 'val1') def test_flavor_summary_from_db_model(self):