api: Add schema validation framework

This is effectively a carbon copy of the code from Nova, Manila, Cinder et
al but modified to work with pecan instead of Routes. We do not use all of
the new code yet, but we will in a future change.

Related-Bug: 2086121

Change-Id: I76c1600036c82ead436cd0fb7e7dee1e34e21907
Signed-off-by: Stephen Finucane <sfinucan@redhat.com>

ironic/api/controllers/v1/shard.py

@@ -18,6 +18,7 @@ from ironic import api
 from ironic.api.controllers.v1 import utils as api_utils
 from ironic.api.controllers.v1 import versions
 from ironic.api import method
+from ironic.api.schemas.v1 import shard as schema
 from ironic.api import validation
 from ironic.common.i18n import _
 
@@ -36,6 +37,8 @@ class ShardController(pecan.rest.RestController):
         min_version=versions.MINOR_82_NODE_SHARD,
         message=_('The API version does not allow shards'),
     )
+    @validation.request_query_schema(schema.index_request_query)
+    @validation.response_body_schema(schema.index_response_body)
     def get_all(self):
         """Retrieve a list of shards.
 
@@ -53,6 +56,6 @@ class ShardController(pecan.rest.RestController):
         min_version=versions.MINOR_82_NODE_SHARD,
         message=_('The API version does not allow shards'),
     )
-    def get_one(self, __):
+    def get_one(self, _):
         """Explicitly do not support getting one."""
         pecan.abort(404)

ironic/api/schemas/v1/shard.py

@@ -0,0 +1,39 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# TODO(stephenfin): Switch additionalProperties to False in a future version
+index_request_query = {
+    'type': 'object',
+    'properties': {},
+    'required': [],
+    'additionalProperties': True,
+}
+
+index_response_body = {
+    'type': 'object',
+    'properties': {
+        'shards': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'count': {'type': 'integer', 'minimum': 1},
+                    'name': {'type': 'string'},
+                },
+                'required': ['count', 'name'],
+                'additionalProperties': False,
+            },
+        },
+    },
+    'required': ['shards'],
+    'additionalProperties': False,
+}

ironic/api/validation/__init__.py

@@ -10,12 +10,17 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+"""API request/response validating middleware."""
+
 import functools
+import inspect
 import typing as ty
 
+from oslo_serialization import jsonutils
 from webob import exc as webob_exc
 
 from ironic import api
+from ironic.api.validation import validators
 from ironic.common.i18n import _
 
 
@@ -70,3 +75,291 @@ def api_version(
         return wrapper
 
     return add_validator
+
+
+class Schemas:
+    """A microversion-aware schema container.
+
+    Allow definition and retrieval of schemas on a microversion-aware basis.
+    """
+
+    def __init__(self) -> None:
+        self._schemas: list[
+            tuple[dict[str, object], ty.Optional[int], ty.Optional[int]]
+        ] = []
+
+    def add_schema(
+        self,
+        schema: tuple[dict[str, object]],
+        min_version: ty.Optional[int],
+        max_version: ty.Optional[int],
+    ) -> None:
+        self._schemas.append((schema, min_version, max_version))
+
+    def __call__(self) -> ty.Optional[dict[str, object]]:
+        for schema, min_version, max_version in self._schemas:
+            if (
+                min_version and not api.request.version.minor >= min_version
+            ) or (
+                max_version and not api.request.version.minor <= max_version
+            ):
+                continue
+
+            return schema
+
+        return None
+
+
+def _schema_validator(
+    schema: ty.Dict[str, ty.Any],
+    target: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[int],
+    max_version: ty.Optional[int],
+    is_body: bool = True,
+):
+    """A helper method to execute JSON Schema Validation.
+
+    This method checks the request version whether matches the specified
+    ``max_version`` and ``min_version``. If the version range matches the
+    request, we validate ``schema`` against ``target``. A failure will result
+    in ``ValidationError`` being raised.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param target: The target to be validated by the schema.
+    :param min_version: An integer indicating the minimum API version
+        ``schema`` applies against.
+    :param max_version: An integer indicating the maximum API version
+        ``schema`` applies against.
+    :param args: Positional arguments which passed into original method.
+    :param kwargs: Keyword arguments which passed into original method.
+    :param is_body: Whether ``target`` is a HTTP request body or not.
+    :returns: None.
+    :raises: ``ValidationError`` if validation fails.
+    """
+    # Only validate against the schema if it lies within
+    # the version range specified. Note that if both min
+    # and max are not specified the validator will always
+    # be run.
+    if (
+        (min_version and api.request.version.minor < min_version)
+        or (max_version and api.request.version.minor > max_version)
+    ):
+        return
+
+    schema_validator = validators.SchemaValidator(schema, is_body=is_body)
+    schema_validator.validate(target)
+
+
+def _extract_parameters(function):
+    sig = inspect.signature(function)
+    params = []
+
+    for param in sig.parameters.values():
+        if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD:
+            if param.name == 'self':  # skip validating self
+                continue
+
+            params.append(param)
+    return params
+
+
+def request_parameter_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[int] = None,
+    max_version: ty.Optional[int] = None,
+):
+    """Decorator for registering a request parameter schema on API methods.
+
+    ``schema`` will be used for validating request parameters just before
+    the API method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: An integer indicating the minimum API version
+        ``schema`` applies against.
+    :param max_version: An integer indicating the maximum API version
+        ``schema`` applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # we need to convert positional arguments to a dict mapping token
+            # name to value so that we have a reference to compare against
+            parameters = _extract_parameters(func)
+            if func.__name__ in ('patch', 'post'):
+                # if this a create or update method, we need to ignore the
+                # request body parameter
+                parameters = parameters[:-1]
+
+            parameters = {
+                p.name: args[i + 1] for i, p in enumerate(parameters)
+                if p.name != '_' and p.default is p.empty
+            }
+            _schema_validator(
+                schema,
+                parameters,
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return func(*args, **kwargs)
+
+        if not hasattr(wrapper, 'request_parameter_schemas'):
+            wrapper.request_parameter_schemas = Schemas()
+
+        wrapper.request_parameter_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator
+
+
+def request_query_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[int] = None,
+    max_version: ty.Optional[int] = None,
+):
+    """Decorator for registering a request query string schema on API methods.
+
+    ``schema`` will be used for validating request query strings just before
+    the API method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: An integer indicating the minimum API version
+        ``schema`` applies against.
+    :param max_version: An integer indicating the maximum API version
+        ``schema`` applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            _schema_validator(
+                schema,
+                kwargs,
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return func(*args, **kwargs)
+
+        if not hasattr(wrapper, 'request_query_schemas'):
+            wrapper.request_query_schemas = Schemas()
+
+        wrapper.request_query_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator
+
+
+def request_body_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[str] = None,
+    max_version: ty.Optional[str] = None,
+):
+    """Decorator for registering a request body schema on API methods.
+
+    ``schema`` will be used for validating the request body just before the API
+    method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: A string indicating the minimum API version ``schema``
+        applies against.
+    :param max_version: A string indicating the maximum API version ``schema``
+        applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            parameters = _extract_parameters(func)
+            if not parameters:
+                # TODO(stephenfin): this would be a better check if we
+                # distinguished between 'create' operations (which should have
+                # at least one parameter, the body) and "update" operations
+                # (which should have at least two, the IDs and the body)
+                raise RuntimeError(
+                    'The ironic.api.method.body decorator must be placed '
+                    'outside the validation helpers to ensure it runs first.'
+                )
+
+            _schema_validator(
+                schema,
+                # The body argument will always be the last one
+                kwargs[parameters[-1].name],
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return func(*args, **kwargs)
+
+        if not hasattr(wrapper, 'request_body_schemas'):
+            wrapper.request_body_schemas = Schemas()
+
+        wrapper.request_body_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator
+
+
+def response_body_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[str] = None,
+    max_version: ty.Optional[str] = None,
+):
+    """Decorator for registering a response body schema on API methods.
+
+    ``schema`` will be used for validating the response body just after the API
+    method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: A string indicating the minimum API version ``schema``
+        applies against.
+    :param max_version: A string indicating the maximum API version ``schema``
+        applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            response = func(*args, **kwargs)
+
+            # FIXME(stephenfin): How is ironic/pecan doing jsonification? The
+            # below will fail on e.g. date-time fields
+
+            # NOTE(stephenfin): If our response is an object, we need to
+            # serialize and deserialize to convert e.g. date-time to strings
+            _body = jsonutils.dumps(response)
+
+            if _body == b'':
+                body = None
+            else:
+                body = jsonutils.loads(_body)
+
+            _schema_validator(
+                schema,
+                body,
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return response
+
+        if not hasattr(wrapper, 'response_body_schemas'):
+            wrapper.response_body_schemas = Schemas()
+
+        wrapper.response_body_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator

ironic/api/validation/validators.py

@@ -0,0 +1,121 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Internal implementation of request/response validating middleware."""
+
+import jsonschema
+from jsonschema import exceptions as jsonschema_exc
+from oslo_utils import timeutils
+from oslo_utils import uuidutils
+
+from ironic.common import exception
+from ironic.common.i18n import _
+
+
+@jsonschema.FormatChecker.cls_checks('date-time')
+def _validate_datetime_format(instance: object) -> bool:
+    # format checks constrain to the relevant primitive type
+    # https://github.com/OAI/OpenAPI-Specification/issues/3148
+    if not isinstance(instance, str):
+        return True
+    try:
+        timeutils.parse_isotime(instance)
+    except ValueError:
+        return False
+    else:
+        return True
+
+
+@jsonschema.FormatChecker.cls_checks('uuid')
+def _validate_uuid_format(instance: object) -> bool:
+    # format checks constrain to the relevant primitive type
+    # https://github.com/OAI/OpenAPI-Specification/issues/3148
+    if not isinstance(instance, str):
+        return True
+
+    return uuidutils.is_uuid_like(instance)
+
+
+class FormatChecker(jsonschema.FormatChecker):
+    """A FormatChecker can output the message from cause exception
+
+    We need understandable validation errors messages for users. When a
+    custom checker has an exception, the FormatChecker will output a
+    readable message provided by the checker.
+    """
+
+    def check(self, param_value, format):
+        """Check whether the param_value conforms to the given format.
+
+        :param param_value: the param_value to check
+        :type: any primitive type (str, number, bool)
+        :param str format: the format that param_value should conform to
+        :raises: :exc:`FormatError` if param_value does not conform to format
+        """
+
+        if format not in self.checkers:
+            return
+
+        # For safety reasons custom checkers can be registered with
+        # allowed exception types. Anything else will fall into the
+        # default formatter.
+        func, raises = self.checkers[format]
+        result, cause = None, None
+
+        try:
+            result = func(param_value)
+        except raises as e:
+            cause = e
+        if not result:
+            msg = '%r is not a %r' % (param_value, format)
+            raise jsonschema_exc.FormatError(msg, cause=cause)
+
+
+class SchemaValidator:
+    """A validator class
+
+    This class is changed from Draft202012Validator to add format checkers for
+    data formats that are common in the Ironic API as well as add better error
+    messages.
+    """
+
+    validator = None
+    validator_org = jsonschema.Draft202012Validator
+
+    def __init__(
+        self, schema, is_body=True
+    ):
+        self.is_body = is_body
+        validator_cls = jsonschema.validators.extend(self.validator_org)
+        format_checker = FormatChecker()
+        try:
+            self.validator = validator_cls(
+                schema, format_checker=format_checker
+            )
+        except Exception:
+            raise
+
+    def validate(self, *args, **kwargs):
+        try:
+            self.validator.validate(*args, **kwargs)
+        except jsonschema.ValidationError as e:
+            error_msg = _('Schema error: %s') % e.message
+            # Sometimes the root message is too generic, try to find a possible
+            # root cause:
+            cause = None
+            current = e
+            while current.context:
+                current = jsonschema.exceptions.best_match(current.context)
+                cause = current.message
+            if cause is not None:
+                error_msg += _('. Possible root cause: %s') % cause
+            raise exception.InvalidParameterValue(error_msg)

ironic/tests/unit/api/validation/test_validators.py

@@ -0,0 +1,46 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ironic.api.validation import validators
+from ironic.common import exception
+from ironic.tests import base as test_base
+
+
+class TestSchemaValidator(test_base.TestCase):
+
+    def test_uuid_format(self):
+        schema = {'type': 'string', 'format': 'uuid'}
+        validator = validators.SchemaValidator(schema)
+
+        # passes
+        validator.validate('d1903ad5-c774-4bfe-8cf4-8e08d8dbb4d3')
+
+        # fails
+        self.assertRaises(
+            exception.InvalidParameterValue,
+            validator.validate,
+            'invalid uuid'
+        )
+
+    def test_datetime_format(self):
+        schema = {'type': 'string', 'format': 'date-time'}
+        validator = validators.SchemaValidator(schema)
+
+        # passes
+        validator.validate('2019-10-12T07:20:50.52Z')
+
+        # fails
+        self.assertRaises(
+            exception.InvalidParameterValue,
+            validator.validate,
+            'invalid date-time'
+        )

releasenotes/notes/add_schema_validation_framework-eaac62cfecb132b0.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds a schema validation framework to the API. This allows for the validation
+    of incoming requests and outgoing responses against a JSON schema, right at
+    the beginning and end of the request processing pipeline.