Implement intermediary file validation

Implements JSON schema validation for intermediary YAML files

Adds tests for intermediary validation

Change-Id: Iaa385d265b027426f8e5f2376462ffb4c0d1d3fa

Pipfile.lock

@@ -128,7 +128,7 @@
         },
         "spyglass-plugin-xls": {
             "git": "https://opendev.org/airship/spyglass-plugin-xls.git",
-            "ref": "269ce7154f88a51e5dd9b883e2f7f15d5326a905"
+            "ref": "3b794d05ffd4731e1b7c4f23bc73a3d73f5ba1c1"
         }
     },
     "develop": {
@@ -198,13 +198,6 @@
             ],
             "version": "==4.5.4"
         },
-        "ddt": {
-            "hashes": [
-                "sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c",
-                "sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3"
-            ],
-            "version": "==1.2.1"
-        },
         "execnet": {
             "hashes": [
                 "sha256:0dd40ad3b960aae93bdad7fe1c3f049bbcc8fba47094655a4301f5b33e906816",
@@ -226,12 +219,6 @@
             ],
             "version": "==2.6.2"
         },
-        "gitdb": {
-            "hashes": [
-                "sha256:a3ebbc27be035a2e874ed904df516e35f4a29a778a764385de09de9e0f139658"
-            ],
-            "version": "==0.6.4"
-        },
         "gitdb2": {
             "hashes": [
                 "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2",
@@ -241,10 +228,10 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:556b64796c5e268b35e3b431a429e813ad54aa178e1baaec2a9ba82e8575a89e",
+                "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276",
-                "sha256:629867ebf609cef21bb9d849039e281e25963fb7d714a2f6bacc1ecce4800293"
+                "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100"
             ],
-            "version": "==3.0.0"
+            "version": "==3.0.2"
         },
         "hacking": {
             "hashes": [
@@ -259,6 +246,7 @@
                 "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8",
                 "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3"
             ],
+            "markers": "python_version < '3.8'",
             "version": "==0.19"
         },
         "mccabe": {
@@ -327,11 +315,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d",
+                "sha256:95b1f6db806e5b1b5b443efeb58984c24945508f93a866c1719e1a507a957d7c",
-                "sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77"
+                "sha256:c3d5020755f70c82eceda3feaf556af9a341334414a8eca521a18f463bcead88"
             ],
             "index": "pypi",
-            "version": "==5.0.1"
+            "version": "==5.1.1"
         },
         "pytest-cov": {
             "hashes": [
@@ -380,12 +368,6 @@
             ],
             "version": "==1.12.0"
         },
-        "smmap": {
-            "hashes": [
-                "sha256:0e2b62b497bd5f0afebc002eda4d90df9d209c30ef257e8673c90a6b5c119d62"
-            ],
-            "version": "==0.9.0"
-        },
         "smmap2": {
             "hashes": [
                 "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde",
@@ -417,10 +399,10 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:6cb2e4c18d22dbbe283d0a0c31bb7d90771a606b2cb3415323eea008eaee6a9d",
+                "sha256:94a6898293d07f84a98add34c4df900f8ec64a570292279f6d91c781d37fd305",
-                "sha256:909fe0d3f7c9151b2df0a2cb53e55bdb7b0d61469353ff7a49fd47b0f0ab9285"
+                "sha256:f6fc312c031f2d2344f885de114f1cb029dfcffd26aa6e57d2ee2296935c4e7d"
             ],
-            "version": "==16.7.2"
+            "version": "==16.7.4"
         },
         "wcwidth": {
             "hashes": [
@@ -439,10 +421,10 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a",
+                "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
-                "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec"
+                "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
             ],
-            "version": "==0.5.2"
+            "version": "==0.6.0"
         }
     }
 }

spyglass/cli.py

@@ -79,7 +79,22 @@ FORCE_OPTION = click.option(
     'force',
     is_flag=True,
     default=False,
-    help="Forces manifests to be written, regardless of undefined data.")
+    help='Forces manifests to be written, regardless of undefined data.')
+
+INTERMEDIARY_SCHEMA_OPTION = click.option(
+    '--intermediary-schema',
+    'intermediary_schema',
+    type=click.Path(exists=True, readable=True, dir_okay=False),
+    default=pkg_resources.resource_filename(
+        'spyglass', "schemas/intermediary_schema.json"),
+    help='Path to the intermediary schema to be used for validation.')
+
+NO_INTERMEDIARY_VALIDATION_OPTION = click.option(
+    '--no-validation',
+    'no_validation',
+    is_flag=True,
+    default=False,
+    help='Skips validation on generated intermediary data.')
 
 
 @click.option(
@@ -135,7 +150,9 @@ def intermediary_processor(plugin_type, **kwargs):
     # Apply design rules to the data
     LOG.info("Apply design rules to the extracted data")
     process_input_ob = ProcessDataSource(
-        kwargs['site_name'], data_extractor.data)
+        kwargs['site_name'], data_extractor.data,
+        kwargs.get('intermediary_schema', None),
+        kwargs.get('no_validation', False))
     return process_input_ob
 
 

spyglass/exceptions.py

@@ -34,8 +34,8 @@ class SpyglassBaseException(Exception):
 class UnsupportedPlugin(SpyglassBaseException):
     """Exception that occurs when a plugin is called that does not exist
 
-    :param plugin_name: name of the specified plugin
+    :keyword plugin_name: name of the specified plugin
-    :param entry_point: the package used to access plugin_name
+    :keyword entry_point: the package used to access plugin_name
     """
     message = (
         '%(plugin_name) was not found in the package %(entry_point) '
@@ -45,10 +45,19 @@ class UnsupportedPlugin(SpyglassBaseException):
 # Data Extractor exceptions
 
 
+class IntermediaryValidationException(SpyglassBaseException):
+    """Exception that occurs when the generated intermediary fails validation
+
+    :keyword errors: list of errors generated by the jsonschema validator
+    """
+    message = (
+        'Intermediary validation failed with the following errors: %(errors)')
+
+
 class InvalidIntermediary(SpyglassBaseException):
     """Exception that occurs when data is missing from the intermediary file
 
-    :param key: dictionary key that Spyglass attempted to access
+    :keyword key: dictionary key that Spyglass attempted to access
     """
     message = '%(key) is not defined in the given intermediary file.'
 
@@ -59,8 +68,8 @@ class InvalidIntermediary(SpyglassBaseException):
 class PathDoesNotExistError(SpyglassBaseException):
     """Exception that occurs when the document or schema path does not exist
 
-    :param file_type: type of the files being accessed, documents or schemas
+    :keyword file_type: type of the files being accessed, documents or schemas
-    :param path: nonexistent path attempted to access
+    :keyword path: nonexistent path attempted to access
     """
     message = '%(file_type) path: %(path) does not exist.'
 
@@ -68,8 +77,8 @@ class PathDoesNotExistError(SpyglassBaseException):
 class UnexpectedFileType(SpyglassBaseException):
     """Exception that occurs when an unexpected file type is given
 
-    :param found_ext: the extension of the file given
+    :keyword found_ext: the extension of the file given
-    :param expected_ext: the extension that was expected for the file
+    :keyword expected_ext: the extension that was expected for the file
     """
     message = (
         'Unexpected file type %(found_ext), '
@@ -82,7 +91,7 @@ class DirectoryEmptyError(SpyglassBaseException):
     This exception can occur when either a directory is empty or if a directory
     does not have any files with the correct file extension.
 
-    :param ext: file extension being searched for
+    :keyword ext: file extension being searched for
-    :param path: path being searched for files of the specified extension
+    :keyword path: path being searched for files of the specified extension
     """
     message = 'No files with %(ext) extension found in document path %(path)'

spyglass/parser/engine.py

@@ -12,29 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
 import json
 import logging
 import os
 import pprint
-import sys
 
-import jsonschema
+from jsonschema import Draft7Validator
 from netaddr import IPNetwork
 from pkg_resources import resource_filename
 import yaml
 
+from spyglass import exceptions
+
 LOG = logging.getLogger(__name__)
 
 
 class ProcessDataSource(object):
-    def __init__(self, region, extracted_data):
+    def __init__(
+            self,
+            region,
+            extracted_data,
+            intermediary_schema=None,
+            no_validation=True):
         # Initialize intermediary and save site type
         self.host_type = {}
         self.sitetype = None
         self.genesis_node = None
         self.network_subnets = None
         self.region_name = region
+        self.no_validation = no_validation
+        if intermediary_schema and not self.no_validation:
+            with open(intermediary_schema, 'r') as loaded_schema:
+                self.intermediary_schema = json.load(loaded_schema)
 
         LOG.info("Loading plugin data source")
         self.data = extracted_data
@@ -74,53 +83,24 @@ class ProcessDataSource(object):
             "Genesis Node Details:\n{}".format(
                 pprint.pformat(self.genesis_node)))
 
-    @staticmethod
+    def _validate_intermediary_data(self):
-    def _validate_intermediary_data(data):
         """Validates the intermediary data before generating manifests.
 
         It checks whether the data types and data format are as expected.
         The method validates this with regex pattern defined for each
         data type.
         """
-        # TODO(ian-pittwood): Implement intermediary validation or remove
 
         LOG.info("Validating Intermediary data")
-        # Performing a deep copy
+        validator = Draft7Validator(self.intermediary_schema)
-        temp_data = copy.deepcopy(data)
+        errors = sorted(
-        # Converting baremetal dict to list.
+            validator.iter_errors(self.data.dict_from_class()),
-        baremetal_list = []
+            key=lambda e: e.path)
-        for rack in temp_data.baremetal:
+        if errors:
-            temp = [{k: v} for k, v in temp_data["baremetal"][rack].items()]
+            raise exceptions.IntermediaryValidationException(errors=errors)
-            baremetal_list = baremetal_list + temp
-
-        temp_data["baremetal"] = baremetal_list
-        schema_dir = resource_filename("spyglass", "schemas/")
-        schema_file = schema_dir + "data_schema.json"
-        json_data = json.loads(json.dumps(temp_data))
-        with open(schema_file, "r") as f:
-            json_schema = json.load(f)
-        try:
-            # Suppressing writing of data2.json. Can use it for debugging
-            # with open('data2.json', 'w') as outfile:
-            #     json.dump(temp_data, outfile, sort_keys=True, indent=4)
-            jsonschema.validate(json_data, json_schema)
-        except jsonschema.exceptions.ValidationError as e:
-            LOG.error("Validation Error")
-            LOG.error("Message:{}".format(e.message))
-            LOG.error("Validator_path:{}".format(e.path))
-            LOG.error("Validator_pattern:{}".format(e.validator_value))
-            LOG.error("Validator:{}".format(e.validator))
-            sys.exit()
-        except jsonschema.exceptions.SchemaError as e:
-            LOG.error("Schema Validation Error!!")
-            LOG.error("Message:{}".format(e.message))
-            LOG.error("Schema:{}".format(e.schema))
-            LOG.error("Validator_value:{}".format(e.validator_value))
-            LOG.error("Validator:{}".format(e.validator))
-            LOG.error("path:{}".format(e.path))
-            sys.exit()
 
         LOG.info("Data validation Passed!")
+        return
 
     def _apply_design_rules(self):
         """Applies design rules from rules.yaml
@@ -309,5 +289,6 @@ class ProcessDataSource(object):
         self._apply_design_rules()
         self._get_genesis_node_details()
         # This will validate the extracted data from different sources.
-        # self._validate_intermediary_data(self.data)
+        if not self.no_validation and self.intermediary_schema:
+            self._validate_intermediary_data()
         return self.data

spyglass/schemas/intermediary_schema.json

@@ -8,7 +8,7 @@
   "type": "object",
   "properties": {
     "baremetal": {
-      "type": "array",
+      "type": "object",
       "items": {
         "type": "object",
         "$ref": "#/definitions/baremetal_list"

tests/conftest.py

@@ -29,6 +29,15 @@ def site_document_data_objects(request):
     request.cls.site_document_data = site_document_data_factory(yaml_data)
 
 
+@pytest.fixture(scope='class')
+def invalid_site_document_data_objects(request):
+    with open(os.path.join(FIXTURE_DIR, 'invalid_intermediary.yaml'),
+              'r') as f:
+        yaml_data = yaml.safe_load(f)
+    request.cls.invalid_site_document_data = site_document_data_factory(
+        yaml_data)
+
+
 @pytest.fixture(scope='class')
 def rules_data(request):
     with open(os.path.join(FIXTURE_DIR, 'rules.yaml'), 'r') as f:

tests/shared/intermediary_schema.json

@@ -0,0 +1,373 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "metadata": {
+    "name": "spyglass/Intermediary/v1"
+  },
+  "title": "All",
+  "description": "All information",
+  "type": "object",
+  "properties": {
+    "baremetal": {
+      "type": "object",
+      "items": {
+        "type": "object",
+        "$ref": "#/definitions/baremetal_list"
+      }
+    },
+    "network": {
+      "type": "object",
+      "properties": {
+        "bgp": {
+          "type": "object",
+          "$ref": "#/definitions/bgp"
+        },
+        "vlan_network_data": {
+          "type": "array",
+          "$ref": "#/definitions/vlan_network_data"
+        }
+      },
+      "required": [
+        "bgp",
+        "vlan_network_data"
+      ]
+    },
+    "site_info": {
+      "type": "object",
+      "$ref": "#/definitions/site_info"
+    },
+    "storage": {
+      "type": "object",
+      "$ref": "#/definitions/storage"
+    }
+  },
+  "required": [
+    "baremetal",
+    "network",
+    "site_info",
+    "storage"
+  ],
+  "definitions": {
+    "baremetal_list": {
+      "type": "object",
+      "patternProperties": {
+        ".*": {
+          "properties": {
+            "ip": {
+              "type": "object",
+              "properties": {
+                "calico": {
+                  "type": "string",
+                  "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
+                },
+                "oam": {
+                  "type": "string",
+                  "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
+                },
+                "oob": {
+                  "type": "string",
+                  "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
+                },
+                "overlay": {
+                  "type": "string",
+                  "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
+                },
+                "pxe": {
+                  "type": "string",
+                  "pattern": "^((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(#CHANGE_ME)$"
+                },
+                "storage": {
+                  "type": "string",
+                  "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
+                }
+             },
+                "required" :[
+                  "calico",
+                  "oam",
+                  "oob",
+                  "overlay",
+                  "pxe",
+                  "storage"
+                  ]
+            },
+            "host_profile": {
+              "description": "Host profile of the host",
+              "type": "string",
+              "pattern": "^([a-zA-Z]+)|(#CHANGE_ME)$"
+            },
+            "type": {
+              "description": "Host profile type:Compute or Controller or genesis ",
+              "type": "string",
+              "pattern": "^(?i)compute|controller|genesis$"
+            }
+          },
+          "required" :[
+            "ip",
+            "host_profile",
+            "type"
+            ]
+        }
+      }
+    },
+    "bgp": {
+      "type": "object",
+      "properties": {
+        "asnumber": {
+          "type": "integer",
+          "pattern": "^[0-9]{1,10}$"
+        },
+        "peer_asnumber": {
+          "type": "integer",
+          "pattern": "^[0-9]{1,10}$"
+        },
+        "peers": {
+          "type": "array",
+          "items": [
+            {
+              "type": "string",
+              "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
+            }
+          ]
+        }
+      },
+      "required": [
+        "asnumber",
+        "peer_asnumber",
+        "peers"
+      ]
+    },
+    "vlan_network_data": {
+      "type": "object",
+      "properties": {
+        "calico": {
+          "type": "object",
+          "properties": {
+            "subnet": {
+              "description": "Subnet address of the network",
+              "type": "array",
+	      "items": {
+		"type": "string",
+              	"pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$"
+		}
+            },
+            "vlan": {
+              "description": "Vlan id of the network",
+              "type": "string",
+              "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$"
+            }
+          },
+          "required": [
+            "subnet",
+            "vlan"
+          ]
+        },
+          "ingress": {
+          "type": "object",
+          "properties": {
+            "subnet": {
+              "description": "Subnet address of the network",
+              "type": "array",
+	      "items": [
+		{
+		"type": "string",
+		"pattern":"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$"
+		}
+		]
+              }
+            },
+          "required": [
+            "subnet"
+          ]
+        },
+          "oam": {
+          "type": "object",
+          "properties": {
+            "subnet": {
+              "description": "Subnet address of the network",
+              "type": "array",
+	      "items": {
+              "type": "string",
+              "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$"
+	      }
+            },
+            "vlan": {
+              "description": "Vlan id of the network",
+              "type": "string",
+              "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$"
+            }
+          },
+          "required": [
+            "subnet",
+	    "vlan"
+          ]
+        },
+          "oob": {
+          "type": "object",
+          "properties": {
+            "subnet": {
+              "description": "Subnet address of the network",
+	      "type": "array",
+	      "items": {
+              "type": "string",
+              "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$"
+	      }
+            },
+            "vlan": {
+              "description": "Vlan id of the network",
+              "type": "string",
+              "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])?$"
+            }
+          },
+          "required": [
+            "subnet"
+          ]
+        },
+          "pxe": {
+          "type": "object",
+          "properties": {
+            "subnet": {
+              "description": "Subnet address of the network",
+	      "type": "array",
+	      "items": {
+              "type": "string",
+              "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$"
+	      }
+            },
+            "vlan": {
+              "description": "Vlan id of the network",
+              "type": "string",
+              "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$"
+            }
+          },
+          "required": [
+            "subnet",
+	    "vlan"
+          ]
+        },
+          "storage": {
+          "type": "object",
+          "properties": {
+            "subnet": {
+              "description": "Subnet address of the network",
+	      "type": "array",
+	      "items": {
+              "type": "string",
+              "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$"
+	     }
+            },
+            "vlan": {
+              "description": "Vlan id of the network",
+              "type": "string",
+              "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$"
+            }
+          },
+          "required": [
+            "subnet",
+	    "vlan"
+          ]
+      }
+      },
+        "required" :[
+          "calico",
+          "ingress",
+          "oam",
+          "oob",
+          "overlay",
+          "pxe",
+          "storage"
+          ]
+    },
+    "site_info": {
+      "type": "object",
+      "properties": {
+        "dns": {
+          "type": "object",
+          "properties": {
+            "servers": {
+              "type": "string",
+              "pattern": "^((((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]),)+)|(((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))+))+$"
+            }
+          }
+        },
+        "ntp": {
+          "type": "object",
+          "properties": {
+            "servers": {
+              "type": "string",
+              "pattern": "^((((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]),)+)|(((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))+))+$"
+            }
+          }
+        },
+        "ldap": {
+          "type": "object",
+          "properties": {
+            "common_name": {
+              "type": "string",
+              "pattern": "\\W+|\\w+"
+            },
+            "subdomain": {
+              "type": "string",
+              "pattern": "(?i)\\w+"
+            },
+            "url": {
+              "type": "string",
+              "pattern": "^\\w+://\\w+.*\\.[a-zA-Z]{2,3}$"
+            }
+          },
+          "required": [
+            "common_name",
+            "subdomain",
+            "url"
+          ]
+        },
+        "country": {
+          "type": "string",
+          "pattern": "(?i)\\w+"
+        },
+        "name": {
+          "type": "string",
+          "pattern": "(?i)\\w+"
+        },
+        "state": {
+          "type": "string",
+          "pattern": "(?i)\\w+"
+        },
+        "sitetype": {
+          "type": "string",
+          "pattern": "(?i)\\w+"
+        },
+        "physical_location_id": {
+          "type": "string",
+          "pattern": "^\\w+"
+        },
+        "domain": {
+          "type": "string",
+          "pattern": "^\\w+.*\\.[a-zA-Z]{2,3}$"
+        }
+      },
+      "required": [
+        "dns",
+        "ntp",
+        "ldap",
+        "country",
+        "name",
+        "state",
+        "sitetype",
+        "physical_location_id",
+        "domain"
+      ]
+    },
+    "storage": {
+      "type": "object",
+      "patternProperties": {
+        "ceph": {
+          "controller": {
+            "osd_count": {
+              "type": "integer",
+              "pattern": "^[0-9]{1,2}$"
+            }
+          }
+        }
+      }
+    }
+  }
+}

tests/shared/invalid_intermediary.yaml

@@ -0,0 +1,216 @@
+baremetal:
+  rack72:
+    cab2r72c12:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.12
+        oam: 10.0.220.12
+        oob: 10.0.220.140
+        overlay: 30.19.0.12
+        pxe: 30.30.4.12
+        storage: 30.31.1.12
+      type: compute
+    cab2r72c13:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.13
+        oam: 10.0.220.13
+        oob: 10.0.220.141
+        overlay: 30.19.0.13
+        pxe: 30.30.4.13
+        storage: 30.31.1.13
+      type: compute
+    cab2r72c14:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.14
+        oam: 10.0.220.14
+        oob: 10.0.220.142
+        overlay: 30.19.0.14
+        pxe: 30.30.4.14
+        storage: 30.31.1.14
+      type: compute
+    cab2r72c15:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.15
+        oam: 10.0.220.15
+        oob: 10.0.220.143
+        overlay: 30.19.0.15
+        pxe: 30.30.4.15
+        storage: 30.31.1.15
+      type: compute
+    cab2r72c16:
+      host_profile: cp-r720
+      ip:
+        calico: 30.29.1.16
+        oam: 10.0.220.16
+        oob: 10.0.220.144
+        overlay: 30.19.0.16
+        pxe: 30.30.4.16
+        storage: 30.31.1.16
+      name: cab2r72c16
+      type: genesis
+    cab2r72c17:
+      host_profile: cp-r720
+      ip:
+        calico: 30.29.1.17
+        oam: 10.0.220.17
+        oob: 10.0.220.145
+        overlay: 30.19.0.17
+        pxe: 30.30.4.17
+        storage: 30.31.1.17
+      type: controller
+  rack73:
+    cab2r73c12:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.18
+        oam: 10.0.220.18
+        oob: 10.0.220.146
+        overlay: 30.19.0.18
+        pxe: 30.30.4.18
+        storage: 30.31.1.18
+      type: compute
+    cab2r73c13:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.19
+        oam: 10.0.220.19
+        oob: 10.0.220.147
+        overlay: 30.19.0.19
+        pxe: 30.30.4.19
+        storage: 30.31.1.19
+      type: compute
+    cab2r73c14:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.20
+        oam: 10.0.220.20
+        oob: 10.0.220.148
+        overlay: 30.19.0.20
+        pxe: 30.30.4.20
+        storage: 30.31.1.20
+      type: compute
+    cab2r73c15:
+      host_profile: dp-r720
+      ip:
+        calico: 30.29.1.21
+        oam: 10.0.220.21
+        oob: 10.0.220.149
+        overlay: 30.19.0.21
+        pxe: 30.30.4.21
+        storage: 30.31.1.21
+      type: compute
+    cab2r73c16:
+      host_profile: cp-r720
+      ip:
+        calico: 30.29.1.22
+        oam: 10.0.220.22
+        oob: 10.0.220.150
+        overlay: 30.19.0.22
+        pxe: 30.30.4.22
+        storage: 30.31.1.22
+      type: controller
+    cab2r73c17:
+      host_profile: cp-r720
+      ip:
+        calico: 30.29.1.23
+        oam: 10.0.220.23
+        oob: 10.0.220.151
+        overlay: 30.19.0.23
+        pxe: 30.30.4.23
+        storage: 30.31.1.23
+      type: controller
+network:
+  bgp:
+    asnumber: 64671
+    ingress_vip: 132.68.226.73
+    peer_asnumber: 64688
+    peers:
+    - 172.29.0.2
+    - 172.29.0.3
+    public_service_cidr: 132.68.226.72/29
+  vlan_network_data:
+    calico:
+      gateway: 30.29.1.1
+      reserved_end: 30.29.1.12
+      reserved_start: 30.29.1.1
+      routes: []
+      static_end: 30.29.1.126
+      static_start: 30.29.1.13
+      subnet:
+      - 30.29.1.0/25
+      vlan: '22'
+    ingress:
+      subnet:
+      - 132.68.226.72/29
+    oob:
+      gateway: 10.0.220.129
+      reserved_end: 10.0.220.138
+      reserved_start: 10.0.220.129
+      routes: []
+      static_end: 10.0.220.158
+      static_start: 10.0.220.139
+      subnet:
+      - 10.0.220.128/27
+      - 10.0.220.160/27
+      - 10.0.220.192/27
+      - 10.0.220.224/27
+    overlay:
+      gateway: 30.19.0.1
+      reserved_end: 30.19.0.12
+      reserved_start: 30.19.0.1
+      routes: []
+      static_end: 30.19.0.126
+      static_start: 30.19.0.13
+      subnet:
+      - 30.19.0.0/25
+      vlan: '24'
+    pxe:
+      dhcp_end: 30.30.4.126
+      dhcp_start: 30.30.4.64
+      gateway: 30.30.4.1
+      reserved_end: 30.30.4.12
+      reserved_start: 30.30.4.1
+      routes: []
+      static_end: 30.30.4.63
+      static_start: 30.30.4.13
+      subnet:
+      - 30.30.4.0/25
+      - 30.30.4.128/25
+      - 30.30.5.0/25
+      - 30.30.5.128/25
+      vlan: '21'
+    storage:
+      gateway: 30.31.1.1
+      reserved_end: 30.31.1.12
+      reserved_start: 30.31.1.1
+      routes: []
+      static_end: 30.31.1.126
+      static_start: 30.31.1.13
+      subnet:
+      - 30.31.1.0/25
+      vlan: '23'
+region_name: test
+site_info:
+  corridor: c1
+  country: SampleCountry
+  dns:
+    servers: 8.8.8.8,8.8.4.4,208.67.222.222
+  domain: atlantafoundry.com
+  ldap:
+    common_name: test
+    domain: example
+    subdomain: test
+    url: ldap://ldap.example.com
+  name: SampleSiteName
+  ntp:
+    servers: 10.10.10.10,20.20.20.20,30.30.30.30
+  physical_location_id: XXXXXX21
+  sitetype: foundry
+  state: New Jersey
+storage:
+  ceph:
+    controller:
+      osd_count: 6

tests/unit/parser/test_engine.py

@@ -19,6 +19,7 @@ from unittest import mock
 from netaddr import IPNetwork
 from pytest import mark
 
+from spyglass import exceptions
 from spyglass.parser.engine import ProcessDataSource
 
 FIXTURE_DIR = os.path.join(
@@ -27,6 +28,7 @@ FIXTURE_DIR = os.path.join(
 
 @mark.usefixtures('tmpdir')
 @mark.usefixtures('site_document_data_objects')
+@mark.usefixtures('invalid_site_document_data_objects')
 @mark.usefixtures('rules_data')
 class TestProcessDataSource(unittest.TestCase):
     REGION_NAME = 'test'
@@ -69,9 +71,20 @@ class TestProcessDataSource(unittest.TestCase):
         obj._get_genesis_node_details()
         self.assertEqual(expected_result, obj.genesis_node)
 
-    @unittest.skip('Not in use.')
     def test__validate_intermediary_data(self):
-        pass
+        schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, schema_path, False)
+        result = obj._validate_intermediary_data()
+        self.assertIsNone(result)
+
+    def test__validate_intermediary_data_invalid(self):
+        schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.invalid_site_document_data, schema_path,
+            False)
+        with self.assertRaises(exceptions.IntermediaryValidationException):
+            obj._validate_intermediary_data()
 
     @mock.patch.object(ProcessDataSource, '_apply_rule_ip_alloc_offset')
     @mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile')