From 1ae47e0b0e094d14982dc333196126c9a4389fbc Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Thu, 17 Apr 2025 12:31:42 +0200 Subject: [PATCH] Switch cli to openstack_types There are lot of changes related to the deduplication caused mostly be lot of data simplifactions done on the cli side for the responses which are not done on openstack_types. This changes were applied manually to the target repository since the changes require also bumping structable version. Some other minor enum sorting and docstring formatting fixes has been applied. Change-Id: Ifc508c54c01bd39af12e3023668e698085829a05 --- codegenerator/common/__init__.py | 3 + codegenerator/common/rust.py | 33 +- codegenerator/model.py | 2 +- codegenerator/rust_cli.py | 497 ++---------------- codegenerator/rust_sdk.py | 16 +- codegenerator/rust_tui.py | 240 --------- codegenerator/rust_types.py | 68 ++- codegenerator/templates/rust_cli/impl.rs.j2 | 87 ++- .../templates/rust_cli/invoke_create.j2 | 2 +- .../templates/rust_cli/invoke_list.j2 | 9 +- .../rust_cli/invoke_list_from_struct.j2 | 24 +- .../templates/rust_cli/invoke_patch.j2 | 9 +- .../templates/rust_cli/invoke_upload.j2 | 2 +- .../templates/rust_cli/response_struct.j2 | 170 ------ codegenerator/templates/rust_macros.j2 | 2 +- codegenerator/templates/rust_tui/impl.rs.j2 | 21 - codegenerator/templates/rust_types/impl.rs.j2 | 118 ++++- codegenerator/tests/unit/test_rust_cli.py | 109 +++- codegenerator/tests/unit/test_rust_sdk.py | 2 - codegenerator/tests/unit/test_rust_types.py | 111 ++++ 20 files changed, 512 insertions(+), 1013 deletions(-) delete mode 100644 codegenerator/templates/rust_cli/response_struct.j2 create mode 100644 codegenerator/tests/unit/test_rust_types.py diff --git a/codegenerator/common/__init__.py b/codegenerator/common/__init__.py index 9544071..0f00ec3 100644 --- a/codegenerator/common/__init__.py +++ b/codegenerator/common/__init__.py @@ -235,6 +235,8 @@ def find_resource_schema( return (schema, None) else: return (schema, None) + # elif schema_type == "string": + # return (schema, None) except Exception as ex: logging.exception( f"Caught exception {ex} during processing of {schema}" @@ -321,6 +323,7 @@ def find_response_schema( in schema.get("properties", []) ) ) + or schema.get("type") == "string" ) ): return schema diff --git a/codegenerator/common/rust.py b/codegenerator/common/rust.py index f8b3c64..44d376f 100644 --- a/codegenerator/common/rust.py +++ b/codegenerator/common/rust.py @@ -273,19 +273,20 @@ class BTreeSet(BaseCombinedType): return imports -class Dictionary(BaseCombinedType): +class Dictionary(BaseCompoundType): base_type: str = "dict" value_type: BasePrimitiveType | BaseCombinedType | BaseCompoundType @property def imports(self): - imports: set[str] = {"std::collections::HashMap"} + imports: set[str] = {"std::collections::BTreeMap"} imports.update(self.value_type.imports) + imports.add("structable::{StructTable, StructTableOptions}") return imports @property def type_hint(self): - return f"HashMap" + return f"BTreeMap" @property def lifetimes(self): @@ -325,6 +326,9 @@ class Struct(BaseCompoundType): additional_fields_type: ( BasePrimitiveType | BaseCombinedType | BaseCompoundType | None ) = None + pattern_properties: ( + BasePrimitiveType | BaseCombinedType | BaseCompoundType | None + ) = None @property def type_hint(self): @@ -382,6 +386,8 @@ class StructFieldResponse(StructField): macros.update(self.data_type.get_serde_macros(self.is_optional)) except Exception: pass + if self.is_optional: + macros.add("default") if self.local_name != self.remote_name: macros.add(f'rename="{self.remote_name}"') if len(macros) > 0: @@ -538,6 +544,7 @@ class StringEnum(BaseCompoundType): lifetimes: set[str] = set() builder_container_macros: str | None = None original_data_type: BaseCompoundType | BaseCompoundType | None = None + allows_arbitrary_value: bool = False @property def derive_container_macros(self) -> str | None: @@ -545,6 +552,8 @@ class StringEnum(BaseCompoundType): @property def serde_container_macros(self) -> str | None: + if self.allows_arbitrary_value: + return "#[serde(untagged)]" return None @property @@ -583,6 +592,7 @@ class StringEnum(BaseCompoundType): class HashMapResponse(Dictionary): """Wrapper around a simple dictionary to implement Display trait""" + # name: str | None = None lifetimes: set[str] = set() @property @@ -592,7 +602,8 @@ class HashMapResponse(Dictionary): @property def imports(self): imports = self.value_type.imports - imports.add("std::collections::HashMap") + imports.add("std::collections::BTreeMap") + imports.add("structable::{StructTable, StructTableOptions}") return imports @@ -607,6 +618,7 @@ class TupleStruct(Struct): imports: set[str] = set() for field in self.tuple_fields: imports.update(field.data_type.imports) + imports.add("structable::{StructTable, StructTableOptions}") return imports @@ -781,7 +793,8 @@ class TypeManager: typ = self._get_one_of_type(type_model) elif isinstance(type_model, model.Dictionary): typ = self.data_type_mapping[model.Dictionary]( - value_type=self.convert_model(type_model.value_type) + name=self.get_model_name(type_model.reference), + value_type=self.convert_model(type_model.value_type), ) elif isinstance(type_model, model.CommaSeparatedList): typ = self.data_type_mapping[model.CommaSeparatedList]( @@ -1001,6 +1014,7 @@ class TypeManager: integer_klass = self.primitive_type_mapping[model.ConstraintInteger] boolean_klass = self.primitive_type_mapping[model.PrimitiveBoolean] dict_klass = self.data_type_mapping[model.Dictionary] + option_klass = self.option_type_class enum_name = type_model.reference.name if type_model.reference else None if string_klass in kinds_classes and number_klass in kinds_classes: # oneOf [string, number] => string @@ -1062,6 +1076,15 @@ class TypeManager: bck = kinds[0].copy() kinds.clear() kinds.append(bck) + elif ( + self.string_enum_class in kinds_classes + and option_klass in kinds_classes + ): + option = next(x for x in kinds if isinstance(x["local"], Option)) + enum = next(x for x in kinds if isinstance(x["local"], StringEnum)) + if option and isinstance(option["local"].item_type, String): + enum["local"].allows_arbitrary_value = True + kinds.remove(option) def set_models(self, models): """Process (translate) ADT models into Rust models""" diff --git a/codegenerator/model.py b/codegenerator/model.py index b9df1b1..d9e7309 100644 --- a/codegenerator/model.py +++ b/codegenerator/model.py @@ -437,7 +437,7 @@ class JsonSchemaParser: # `"type": "object", "pattern_properties": ...` if len(list(pattern_props.values())) == 1: obj = Dictionary( - value_type=list(pattern_props.values())[0] + name=name, value_type=list(pattern_props.values())[0] ) else: obj = Struct(pattern_properties=pattern_props) diff --git a/codegenerator/rust_cli.py b/codegenerator/rust_cli.py index 40316f1..4a7e650 100644 --- a/codegenerator/rust_cli.py +++ b/codegenerator/rust_cli.py @@ -69,47 +69,18 @@ class SecretString(String): pass -class IntString(common.BasePrimitiveType): - """CLI Integer or String""" - - imports: set[str] = {"openstack_sdk::types::IntString"} - type_hint: str = "IntString" - clap_macros: set[str] = set() - - -class NumString(common.BasePrimitiveType): - """CLI Number or String""" - - imports: set[str] = {"openstack_sdk::types::NumString"} - type_hint: str = "NumString" - clap_macros: set[str] = set() - - -class BoolString(common.BasePrimitiveType): - """CLI Boolean or String""" - - imports: set[str] = {"openstack_sdk::types::BoolString"} - type_hint: str = "BoolString" - clap_macros: set[str] = set() - - -class VecString(common.BasePrimitiveType): - """CLI Vector of strings""" - - imports: set[str] = {"crate::common::VecString"} - type_hint: str = "VecString" - clap_macros: set[str] = set() - - class JsonValue(common_rust.JsonValue): """Arbitrary JSON value""" - clap_macros: set[str] = {'value_name="JSON"', "value_parser=parse_json"} + clap_macros: set[str] = { + 'value_name="JSON"', + "value_parser=crate::common::parse_json", + } original_data_type: BaseCombinedType | BaseCompoundType | None = None @property def imports(self): - imports: set[str] = {"crate::common::parse_json", "serde_json::Value"} + imports: set[str] = {"serde_json::Value"} if self.original_data_type and isinstance( self.original_data_type, common_rust.Dictionary ): @@ -240,108 +211,6 @@ class EnumGroupStruct(common_rust.Struct): reference: model.Reference | None = None -class StructFieldResponse(common_rust.StructField): - """Response Structure Field""" - - @property - def type_hint(self): - typ_hint = self.data_type.type_hint - if self.is_optional and not typ_hint.startswith("Option<"): - typ_hint = f"Option<{typ_hint}>" - return typ_hint - - @property - def serde_macros(self): - macros = set() - if self.local_name != self.remote_name: - macros.add(f'rename="{self.remote_name}"') - return f"#[serde({', '.join(sorted(macros))})]" - - def get_structable_macros( - self, - struct: "StructResponse", - service_name: str, - resource_name: str, - operation_type: str, - ): - macros = set() - if self.is_optional or self.data_type.type_hint.startswith("Option<"): - macros.add("optional") - if self.local_name != self.remote_name: - macros.add(f'title="{self.remote_name}"') - # Fully Qualified Attribute Name - fqan: str = ".".join( - [service_name, resource_name, self.remote_name] - ).lower() - # Check the known alias of the field by FQAN - alias = common.FQAN_ALIAS_MAP.get(fqan) - if operation_type in ["list", "list_from_struct"]: - if ( - "id" in struct.fields.keys() - and not ( - self.local_name in BASIC_FIELDS or alias in BASIC_FIELDS - ) - ) or ( - "id" not in struct.fields.keys() - and (self.local_name not in list(struct.fields.keys())[-10:]) - and not ( - self.local_name in BASIC_FIELDS or alias in BASIC_FIELDS - ) - ): - # Only add "wide" flag if field is not in the basic fields AND - # there is at least "id" field existing in the struct OR the - # field is not in the first 10 - macros.add("wide") - if ( - self.local_name == "state" - and "status" not in struct.fields.keys() - ): - macros.add("status") - elif ( - self.local_name == "operating_status" - and "status" not in struct.fields.keys() - ): - macros.add("status") - if self.data_type.type_hint in [ - "Value", - "Option", - "Vec", - "Option>", - ]: - macros.add("pretty") - return f"#[structable({', '.join(sorted(macros))})]" - - -class StructResponse(common_rust.Struct): - field_type_class_: Type[common_rust.StructField] = StructFieldResponse - - @property - def imports(self): - imports: set[str] = {"serde::Deserialize"} - for field in self.fields.values(): - imports.update(field.data_type.imports) - # In difference to the SDK and Input we do not currently handle - # additional_fields of the struct in response - # if self.additional_fields_type: - # imports.add("std::collections::BTreeMap") - # imports.update(self.additional_fields_type.imports) - return imports - - -class TupleStruct(common_rust.Struct): - """Rust tuple struct without named fields""" - - base_type: str = "struct" - tuple_fields: list[common_rust.StructField] = [] - - @property - def imports(self): - imports: set[str] = set() - for field in self.tuple_fields: - imports.update(field.data_type.imports) - return imports - - class DictionaryInput(common_rust.Dictionary): lifetimes: set[str] = set() original_data_type: BaseCompoundType | BaseCompoundType | None = None @@ -392,37 +261,13 @@ class ArrayInput(common_rust.Array): macros: set[str] = {"long", "action=clap::ArgAction::Append"} macros.update(self.item_type.clap_macros) if isinstance(self.item_type, ArrayInput): - macros.add("value_parser=parse_json") + macros.add("value_parser=crate::common::parse_json") macros.add( f'value_name="[{self.item_type.item_type.type_hint}] as JSON"' ) return macros -class ArrayResponse(common_rust.Array): - """Vector of data for the Reponse - - in the reponse need to be converted to own type to implement Display""" - - @property - def type_hint(self): - return f"Vec{self.item_type.type_hint}" - - -class HashMapResponse(common_rust.Dictionary): - lifetimes: set[str] = set() - - @property - def type_hint(self): - return f"HashMapString{self.value_type.type_hint.replace('<', '').replace('>', '')}" - - @property - def imports(self): - imports = self.value_type.imports - imports.add("std::collections::HashMap") - return imports - - class CommaSeparatedList(common_rust.CommaSeparatedList): @property def type_hint(self): @@ -627,7 +472,8 @@ class RequestTypeManager(common_rust.TypeManager): original_data_type = self.convert_model(type_model.value_type) typ = JsonValue( original_data_type=DictionaryInput( - value_type=original_data_type + name=self.get_model_name(type_model.reference), + value_type=original_data_type, ) ) else: @@ -638,6 +484,7 @@ class RequestTypeManager(common_rust.TypeManager): type_model.value_type ) typ = DictionaryInput( + name=self.get_model_name(type_model.reference), description=type_model.value_type.description, value_type=JsonValue( original_data_type=original_data_type @@ -821,198 +668,6 @@ class RequestTypeManager(common_rust.TypeManager): self.parameters[k] = param -class ResponseTypeManager(common_rust.TypeManager): - primitive_type_mapping: dict[ - Type[model.PrimitiveType], Type[BasePrimitiveType] - ] = { - model.PrimitiveString: common_rust.String, - model.ConstraintString: common_rust.String, - } - - data_type_mapping = { - model.Struct: StructResponse, - model.Array: JsonValue, - model.Dictionary: JsonValue, - } - - def get_model_name(self, model_ref: model.Reference | None) -> str: - """Get the localized model type name - - In order to avoid collision between structures in request and - response we prefix all types with `Response` - :returns str: Type name - """ - if not model_ref: - return "Response" - return "Response" + "".join( - x.capitalize() - for x in re.split(common.SPLIT_NAME_RE, model_ref.name) - ) - - def convert_model( - self, type_model: model.PrimitiveType | model.ADT | model.Reference - ) -> BasePrimitiveType | BaseCombinedType | BaseCompoundType: - """Get local destination type from the ModelType""" - model_ref: model.Reference | None = None - typ: BasePrimitiveType | BaseCombinedType | BaseCompoundType | None = ( - None - ) - if isinstance(type_model, model.Reference): - model_ref = type_model - type_model = self._get_adt_by_reference(model_ref) - elif isinstance(type_model, model.ADT): - # Direct composite type - model_ref = type_model.reference - - # CLI response PRE hacks - if isinstance(type_model, model.Array): - item_type = type_model.item_type - if isinstance(item_type, String): - # Array of string is replaced by `VecString` type - typ = VecString() - elif ( - model_ref - and model_ref.name == "links" - and model_ref.type == model.Array - ): - # Array of "links" is replaced by Json Value - typ = common_rust.JsonValue() - self.ignored_models.append(type_model.item_type) - elif ( - isinstance(item_type, model.Reference) - and type_model.item_type.type == model.Struct - ): - # Array of complex Structs is replaced on output by Json Value - typ = common_rust.JsonValue() - self.ignored_models.append(item_type) - if typ: - if model_ref: - self.refs[model_ref] = typ - else: - # Not hacked anything, invoke superior method - typ = super().convert_model(type_model) - - # POST hacks - if typ and isinstance(typ, common_rust.StringEnum): - # There is no sense of Enum in the output. Convert to the plain - # string - typ = String( - description=common_rust.sanitize_rust_docstrings( - typ.description - ) - ) - if ( - typ - and isinstance(typ, ArrayResponse) - and isinstance(typ.item_type, common_rust.Enum) - ): - # Array of complex Enums is replaced on output by Json Value - self.ignored_models.append(typ.item_type) - typ = common_rust.JsonValue() - return typ - - def _simplify_oneof_combinations(self, type_model, kinds): - """Simplify certain known oneOf combinations""" - kinds_classes = [x["class"] for x in kinds] - if ( - common_rust.String in kinds_classes - and common_rust.Number in kinds_classes - ): - # oneOf [string, number] => NumString - kinds.clear() - kinds.append({"local": NumString(), "class": NumString}) - elif ( - common_rust.String in kinds_classes - and common_rust.Integer in kinds_classes - ): - # oneOf [string, integer] => NumString - kinds.clear() - kinds.append({"local": IntString(), "class": IntString}) - elif ( - common_rust.String in kinds_classes - and common_rust.Boolean in kinds_classes - ): - # oneOf [string, boolean] => String - kinds.clear() - kinds.append({"local": BoolString(), "class": BoolString}) - super()._simplify_oneof_combinations(type_model, kinds) - - def _get_struct_type(self, type_model: model.Struct) -> common_rust.Struct: - """Convert model.Struct into Rust `Struct`""" - struct_class = self.data_type_mapping[model.Struct] - mod = struct_class( - name=self.get_model_name(type_model.reference), - description=common_rust.sanitize_rust_docstrings( - type_model.description - ), - ) - field_class = mod.field_type_class_ - for field_name, field in type_model.fields.items(): - is_nullable: bool = False - field_data_type = self.convert_model(field.data_type) - if isinstance(field_data_type, self.option_type_class): - # Unwrap Option into "is_nullable" NOTE: but perhaps - # Option