openstack/codegenerator
Code Issues Proposed changes

Enable generation of manila openapi

Browse Source 
- Add Manila stub
- Add support for newer decorators with response info
- add new jobs for building spec

Change-Id: I3aa578af4ca50297ad0f3860f69e33df59ac6f22
This commit is contained in:
Artem Goncharov 2024-06-17 18:21:44 +02:00
parent 3ac8000fcb
commit 2e45767a49
7 changed files with 295 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

148
codegenerator/openapi/base.py
View File

@ -17,7 +17,7 @@ import importlib
import inspect
import logging
from pathlib import Path
from typing import Any
from typing import Any, Callable
import re
from codegenerator.common.schema import ParameterSchema
@ -365,25 +365,28 @@ class OpenStackServerSourceBase:
for action, op_name in controller_actions.items():
logging.info("Action %s: %s", action, op_name)
(start_version, end_version) = (None, None)
action_impls: list[tuple[Callable, str | None, str | None]] = (
[]
)
if isinstance(op_name, str):
# wsgi action value is a string
if op_name in versioned_methods:
# ACTION with version bounds
if len(versioned_methods[op_name]) > 1:
raise RuntimeError(
"Multiple versioned methods for action %s",
action,
)
for ver_method in versioned_methods[op_name]:
start_version = ver_method.start_version
end_version = ver_method.end_version
func = ver_method.func
logging.info("Versioned action %s", func)
# operation_id += f"[{op_name}]"
action_impls.append(
(
ver_method.func,
ver_method.start_version,
ver_method.end_version,
)
)
logging.info(
"Versioned action %s", ver_method.func
)
elif hasattr(contr, op_name):
# ACTION with no version bounds
func = getattr(contr, op_name)
# operation_id += f"[{op_name}]"
action_impls.append((func, None, None))
logging.info("Unversioned action %s", func)
else:
logging.error(
@ -405,15 +408,20 @@ class OpenStackServerSourceBase:
if key and key in versioned_methods:
# ACTION with version bounds
if len(versioned_methods[key]) > 1:
raise RuntimeError(
"Multiple versioned methods for action %s",
action,
logging.warn(
f"There are multiple callables for action {key} instead of multiple bodies"
)
for ver_method in versioned_methods[key]:
start_version = ver_method.start_version
end_version = ver_method.end_version
func = ver_method.func
logging.info("Versioned action %s", func)
action_impls.append(
(
ver_method.func,
ver_method.start_version,
ver_method.end_version,
)
)
logging.info(
"Versioned action %s", ver_method.func
)
elif slf and key:
vm = getattr(slf, "versioned_methods", None)
if vm and key in vm:
@ -424,12 +432,18 @@ class OpenStackServerSourceBase:
action,
)
for ver_method in vm[key]:
start_version = ver_method.start_version
end_version = ver_method.end_version
func = ver_method.func
logging.info("Versioned action %s", func)
action_impls.append(
(
ver_method.func,
ver_method.start_version,
ver_method.end_version,
)
)
logging.info(
"Versioned action %s", ver_method.func
)
else:
func = op_name
action_impls.append((op_name, None, None))
# Get the path/op spec only when we have
# something to fill in
@ -442,19 +456,20 @@ class OpenStackServerSourceBase:
operation_spec.tags.extend(operation_tags)
operation_spec.tags = list(set(operation_spec.tags))
self.process_operation(
func,
openapi_spec,
operation_spec,
path_resource_names,
controller=controller,
operation_name=action,
method=method,
start_version=start_version,
end_version=end_version,
mode="action",
path=path,
)
for func, start_version, end_version in action_impls:
self.process_operation(
func,
openapi_spec,
operation_spec,
path_resource_names,
controller=controller,
operation_name=action,
method=method,
start_version=start_version,
end_version=end_version,
mode="action",
path=path,
)
elif framework == "pecan":
if callable(controller):
func = controller
@ -534,6 +549,22 @@ class OpenStackServerSourceBase:
operation_name,
func,
)
# New decorators start having explicit null ApiVersion instead of being null
if (
start_version
and not isinstance(start_version, str)
and self._api_ver_major(start_version) in [0, None]
and self._api_ver_minor(start_version) in [0, None]
):
start_version = None
if (
end_version
and not isinstance(end_version, str)
and self._api_ver_major(end_version) in [0, None]
and self._api_ver_minor(end_version) in [0, None]
):
end_version = None
deser_schema = None
deser = getattr(controller, "deserializer", None)
if deser:
@ -583,8 +614,12 @@ class OpenStackServerSourceBase:
start_version.get_string()
)
if mode != "action" and end_version:
if end_version.ver_major == 0:
if (
mode != "action"
and end_version
and self._api_ver_major(end_version)
):
if self._api_ver_major(end_version) == 0:
operation_spec.openstack.pop("max-ver", None)
operation_spec.deprecated = None
else:
@ -618,7 +653,11 @@ class OpenStackServerSourceBase:
closure = inspect.getclosurevars(f)
closure_locals = closure.nonlocals
min_ver = closure_locals.get("min_version", start_version)
if min_ver and not isinstance(min_ver, str):
min_ver = min_ver.get_string()
max_ver = closure_locals.get("max_version", end_version)
if max_ver and not isinstance(max_ver, str):
max_ver = max_ver.get_string()
if "errors" in closure_locals:
expected_errors = closure_locals["errors"]
@ -631,9 +670,14 @@ class OpenStackServerSourceBase:
]
elif isinstance(expected_errors, int):
expected_errors = [str(expected_errors)]
if "request_body_schema" in closure_locals:
if "request_body_schema" in closure_locals or hasattr(
f, "_request_body_schema"
):
# Body type is known through method decorator
obj = closure_locals["request_body_schema"]
obj = closure_locals.get(
"request_body_schema",
getattr(f, "_request_body_schema", {}),
)
if obj.get("type") in ["object", "array"]:
# We only allow object and array bodies
# To prevent type name collision keep module name part of the name
@ -666,8 +710,22 @@ class OpenStackServerSourceBase:
ref_name = f"#/components/schemas/{typ_name}"
body_schemas.append(ref_name)
if "query_params_schema" in closure_locals:
obj = closure_locals["query_params_schema"]
if "response_body_schema" in closure_locals or hasattr(
f, "_response_body_schema"
):
# Response type is known through method decorator - PERFECT
obj = closure_locals.get(
"response_body_schema",
getattr(f, "_response_body_schema", {}),
)
ser_schema = obj
if "query_params_schema" in closure_locals or hasattr(
f, "_request_query_schema"
):
obj = closure_locals.get(
"query_params_schema",
getattr(f, "_request_query_schema", {}),
)
query_params_versions.append((obj, min_ver, max_ver))
f = f.__wrapped__
@ -704,7 +762,9 @@ class OpenStackServerSourceBase:
if query_params_versions:
so = sorted(
query_params_versions,
key=lambda d: d[1].split(".") if d[1] else (0, 0),
key=lambda d: (
tuple(map(int, d[1].split("."))) if d[1] else (0, 0)
),
)
for data, min_ver, max_ver in so:
self.process_query_parameters(

145
codegenerator/openapi/manila.py Normal file
View File

@ -0,0 +1,145 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from multiprocessing import Process
from pathlib import Path
from ruamel.yaml.scalarstring import LiteralScalarString
from codegenerator.common.schema import (
SpecSchema,
)
from codegenerator.openapi.base import OpenStackServerSourceBase
from codegenerator.openapi.utils import merge_api_ref_doc
class ManilaGenerator(OpenStackServerSourceBase):
URL_TAG_MAP = {
"/versions": "version",
}
def _api_ver_major(self, ver):
return ver._ver_major
def _api_ver_minor(self, ver):
return ver._ver_minor
def _api_ver(self, ver):
return (ver._ver_major, ver._ver_minor)
def _generate(self, target_dir, args):
import fixtures
from oslo_config import cfg
from oslo_config import fixture as config_fixture
from oslo_concurrency import lockutils
from manila.api.openstack import api_version_request
from manila.api.v2 import router
from manila import rpc
# from placement import handler
from manila.common import config
from manila import coordination
self.api_version = api_version_request._MAX_API_VERSION
self.min_api_version = api_version_request._MIN_API_VERSION
CONF = config.CONF
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path, group="oslo_concurrency")
self.fixture.config(
disable_process_locking=True, group="oslo_concurrency"
)
rpc.init(CONF)
CONF.set_override(
"backend_url", "file://" + lock_path, group="coordination"
)
coordination.LOCK_COORDINATOR.start()
# config = cfg.ConfigOpts()
# conf_fixture = self.useFixture(config_fixture.Config(config))
# conf.register_opts(conf_fixture.conf)
# handler = handler.PlacementHandler(config=conf_fixture.conf)
self.router = router.APIRouter()
work_dir = Path(target_dir)
work_dir.mkdir(parents=True, exist_ok=True)
impl_path = Path(
work_dir,
"openapi_specs",
"shared_file_system",
f"v{self.api_version}.yaml",
)
impl_path.parent.mkdir(parents=True, exist_ok=True)
openapi_spec = self.load_openapi(impl_path)
if not openapi_spec:
openapi_spec = SpecSchema(
info=dict(
title="OpenStack Shared-File-System API",
description=LiteralScalarString(
"Shared File System API provided by Manila service"
),
version=self.api_version,
),
openapi="3.1.0",
security=[{"ApiKeyAuth": []}],
components=dict(
securitySchemes={
"ApiKeyAuth": {
"type": "apiKey",
"in": "header",
"name": "X-Auth-Token",
}
},
),
)
for route in self.router.map.matchlist:
if route.routepath.endswith(".:(format)"):
continue
if route.routepath.startswith("/{project"):
continue
# if not route.routepath.startswith("/resource-lock"):
# continue
self._process_route(route, openapi_spec)
self._sanitize_param_ver_info(openapi_spec, self.min_api_version)
if args.api_ref_src:
merge_api_ref_doc(
openapi_spec,
args.api_ref_src,
allow_strip_version=False,
)
self.dump_openapi(openapi_spec, impl_path, args.validate)
lnk = Path(impl_path.parent, "v2.yaml")
lnk.unlink(missing_ok=True)
lnk.symlink_to(impl_path.name)
return impl_path
def generate(self, target_dir, args):
proc = Process(target=self._generate, args=[target_dir, args])
proc.start()
proc.join()
if proc.exitcode != 0:
raise RuntimeError("Error generating Manila OpenAPI schema")

7
codegenerator/openapi_spec.py
View File

@ -63,6 +63,11 @@ class OpenApiSchemaGenerator(BaseGenerator):
PlacementGenerator().generate(target_dir, args)
def generate_manila(self, target_dir, args):
from codegenerator.openapi.manila import ManilaGenerator
ManilaGenerator().generate(target_dir, args)
def generate(
self, res, target_dir, openapi_spec=None, operation_id=None, args=None
):
@ -85,6 +90,8 @@ class OpenApiSchemaGenerator(BaseGenerator):
self.generate_neutron(target_dir, args)
elif args.service_type == "placement":
self.generate_placement(target_dir, args)
elif args.service_type == "shared-file-system":
self.generate_manila(target_dir, args)
else:
raise RuntimeError(
"Service type %s is not supported", args.service_type

2
setup.cfg
View File

@ -43,6 +43,8 @@ network =
neutron-vpnaas>=23.0
placement =
openstack-placement>=10.0
shared-file-system =
manila>=18.0
[mypy]
show_column_numbers = true

4
tools/generate_openapi_specs.sh
View File

@ -30,3 +30,7 @@ if [ -z "$1" -o "$1" = "placement" ]; then
openstack-codegenerator --work-dir wrk --target openapi-spec --service-type placement --api-ref-src ${API_REF_BUILD_ROOT}/placement/api-ref/build/html/index.html --validate
sed -i "s/(?expanded=delete-resource-provider-inventories-detail#delete-resource-provider-inventories)//" wrk/openapi_specs/placement/v1.yaml
fi
if [ -z "$1" -o "$1" = "shared-file-system" ]; then
openstack-codegenerator --work-dir wrk --target openapi-spec --service-type shared-file-system --api-ref-src ${API_REF_BUILD_ROOT}/manila/api-ref/build/html/index.html --validate
fi

31
zuul.d/openapi.yaml
View File

@ -239,6 +239,35 @@
project: "opendev.org/openstack/placement"
path: "/api-ref/build/html/index.html"
- job:
name: codegenerator-openapi-shared-file-system-tips
parent: codegenerator-openapi-tips-base
description: |
Generate OpenAPI spec for Manila
required-projects:
- name: openstack/manila
vars:
openapi_service: shared-file-system
install_additional_projects:
- project: "opendev.org/openstack/manila"
name: "."
- job:
name: codegenerator-openapi-shared-file-system-tips-with-api-ref
parent: codegenerator-openapi-shared-file-system-tips
description: |
Generate OpenAPI spec for Manila consuming API-REF
required-projects:
- name: openstack/manila
pre-run:
- playbooks/openapi/pre-api-ref.yaml
vars:
codegenerator_api_ref:
project: "opendev.org/openstack/manila"
path: "/api-ref/build/html/index.html"
- job:
name: codegenerator-tox-publish-openapi-specs
parent: opendev-tox-docs
@ -259,6 +288,8 @@
soft: true
- name: codegenerator-openapi-placement-tips-with-api-ref
soft: true
- name: codegenerator-openapi-shared-file-system-tips-with-api-ref
soft: true
pre-run:
- playbooks/openapi/fetch.yaml
vars:

2
zuul.d/project.yaml
View File

@ -13,6 +13,7 @@
- codegenerator-openapi-load-balancing-tips-with-api-ref
- codegenerator-openapi-network-tips-with-api-ref
- codegenerator-openapi-placement-tips-with-api-ref
- codegenerator-openapi-shared-file-system-tips-with-api-ref
- codegenerator-tox-publish-openapi-specs
gate:
jobs:
@ -25,4 +26,5 @@
- codegenerator-openapi-load-balancing-tips-with-api-ref
- codegenerator-openapi-network-tips-with-api-ref
- codegenerator-openapi-placement-tips-with-api-ref
- codegenerator-openapi-shared-file-system-tips-with-api-ref
- codegenerator-tox-publish-openapi-specs