Add CIDR validation for MAAS

MAAS only accepts CIDR IPs that do not have host bits set otherwise
MAAS sees the CIDR as a second network. This commit adds a Drydock
validation that checks if the CIDR has host bits and also suggests
which CIDR to use if the provided one is not acceptable to MAAS.

Change-Id: Ib6d4d8277d0e1634524426a08e138e39fb37f14b

python/drydock_provisioner/orchestrator/validations/cidr_validity.py

@@ -0,0 +1,47 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import ipaddress
+
+from drydock_provisioner.orchestrator.validations.validators import Validators
+
+
+class CidrValidity(Validators):
+    def __init__(self):
+        super().__init__('CIDR Validity', 'DD2006')
+
+    def run_validation(self, site_design, orchestrator=None):
+        """
+        Ensures that the network CIDR provided is acceptable to MAAS which
+        means the CIDR should not have host bits set
+        """
+        network_list = site_design.networks or []
+
+        if not network_list:
+            msg = 'No networks found.'
+            self.report_warn(
+                msg, [],
+                'Site design likely incomplete without defined networks')
+        else:
+            for net in network_list:
+                # Verify that the CIDR does not have host bits set
+                try:
+                    ipaddress.ip_network(net.cidr)
+                except ValueError as e:
+                    if str(e) == (net.cidr + " has host bits set"):
+                        msg = 'The provided CIDR %s has host bits set' % net.cidr
+                        valid_cidr = ipaddress.ip_network(net.cidr, strict=False)
+                        self.report_error(
+                                    msg, [net.doc_ref],
+                                    "Provide a CIDR acceptable by MAAS: %s" % str(valid_cidr))
+        return

python/drydock_provisioner/orchestrator/validations/validator.py

@@ -18,6 +18,7 @@ import drydock_provisioner.objects.fields as hd_fields
 from drydock_provisioner.objects.validation import Validation
 
 from drydock_provisioner.orchestrator.validations.boot_storage_rational import BootStorageRational
+from drydock_provisioner.orchestrator.validations.cidr_validity import CidrValidity
 from drydock_provisioner.orchestrator.validations.hugepages_validity import HugepagesValidity
 from drydock_provisioner.orchestrator.validations.ip_locality_check import IpLocalityCheck
 from drydock_provisioner.orchestrator.validations.mtu_rational import MtuRational
@@ -84,6 +85,7 @@ class Validator():
 
 rule_set = [
     BootStorageRational(),
+    CidrValidity(),
     HugepagesValidity(),
     IpLocalityCheck(),
     MtuRational(),

python/tests/unit/test_validation_rule_network_cidr.py

@@ -0,0 +1,67 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test Validation Rule Network CIDR"""
+
+import re
+import logging
+
+from drydock_provisioner.orchestrator.orchestrator import Orchestrator
+from drydock_provisioner.orchestrator.validations.cidr_validity import CidrValidity
+
+LOG = logging.getLogger(__name__)
+
+
+class TestNetworkCidr(object):
+    def test_valid_network_cidr(self, mocker, deckhand_ingester, drydock_state,
+                                input_files):
+
+        input_file = input_files.join("validation.yaml")
+        design_ref = "file://%s" % str(input_file)
+
+        orch = Orchestrator(
+            state_manager=drydock_state, ingester=deckhand_ingester)
+
+        status, site_design = Orchestrator.get_effective_site(orch, design_ref)
+
+        validator = CidrValidity()
+        message_list = validator.execute(site_design, orchestrator=orch)
+        msg = message_list[0].to_dict()
+
+        assert msg.get('error') is False
+        assert len(message_list) == 1
+
+    def test_invalid_network_cidr(self, mocker, deckhand_ingester,
+                                  drydock_state, input_files):
+
+        input_file = input_files.join("invalid_network_cidr.yaml")
+        design_ref = "file://%s" % str(input_file)
+
+        orch = Orchestrator(
+            state_manager=drydock_state, ingester=deckhand_ingester)
+
+        status, site_design = Orchestrator.get_effective_site(orch, design_ref)
+
+        validator = CidrValidity()
+        message_list = validator.execute(site_design, orchestrator=orch)
+        msg = message_list[0].to_dict()
+
+        regex_diagnostic = re.compile('Provide a CIDR acceptable by MAAS: .+')
+        regex_message = re.compile('The provided CIDR .+ has host bits set')
+
+        assert len(message_list) >= 1
+        assert msg.get('error') is True
+        assert any([
+            regex_diagnostic.search(msg.get('diagnostic')),
+            regex_message.search(msg.get('message'))
+                  ])

python/tests/yaml_samples/invalid_network_cidr.yaml

@@ -0,0 +1,449 @@
+#Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+####################
+#
+# bootstrap_seed.yaml - Site server design definition for physical layer
+#
+####################
+# version the schema in this file so consumers can rationally parse it
+---
+schema: 'drydock/Region/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: 'sitename'
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  tag_definitions:
+    - tag: 'test'
+      definition_type: 'lshw_xpath'
+      definition: "//node[@id=\"display\"]/'clock units=\"Hz\"' > 1000000000"
+  authorized_keys:
+    - |
+      ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDENeyO5hLPbLLQRZ0oafTYWs1ieo5Q+XgyZQs51Ju
+      jDGc8lKlWsg1/6yei2JewKMgcwG2Buu1eqU92Xn1SvMZLyt9GZURuBkyjcfVc/8GiU5QP1Of8B7CV0c
+      kfUpHWYJ17olTzT61Hgz10ioicBF6cjgQrLNcyn05xoaJHD2Vpf8Unxzi0YzA2e77yRqBo9jJVRaX2q
+      wUJuZrzb62x3zw8Knz6GGSZBn8xRKLaw1SKFpd1hwvL62GfqX5ZBAT1AYTZP1j8GcAoK8AFVn193SEU
+      vjSdUFa+RNWuJhkjBRfylJczIjTIFb5ls0jpbA3bMA9DE7lFKVQl6vVwFmiIVBI1 samplekey
+---
+schema: 'drydock/NetworkLink/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: oob
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  bonding:
+    mode: disabled
+  mtu: 1500
+  linkspeed: 100full
+  trunking:
+    mode: disabled
+    default_network: oob
+  allowed_networks:
+    - oob
+---
+schema: 'drydock/NetworkLink/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: pxe
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  bonding:
+    mode: disabled
+  mtu: 1500
+  linkspeed: auto
+  trunking:
+    mode: disabled
+    default_network: pxe
+  allowed_networks:
+    - pxe
+---
+schema: 'drydock/NetworkLink/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: gp
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  bonding:
+    mode: 802.3ad
+    hash: layer3+4
+    peer_rate: slow
+  mtu: 9000
+  linkspeed: auto
+  trunking:
+    mode: 802.1q
+    default_network: mgmt
+  allowed_networks:
+    - public
+    - private
+    - mgmt
+---
+schema: 'drydock/Rack/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: rack1
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  tor_switches:
+    switch01name:
+      mgmt_ip: 1.1.1.1
+      sdn_api_uri: polo+https://api.sdn.example.com/switchmgmt?switch=switch01name
+    switch02name:
+      mgmt_ip: 1.1.1.2
+      sdn_api_uri: polo+https://api.sdn.example.com/switchmgmt?switch=switch02name
+  location:
+    clli: HSTNTXMOCG0
+    grid: EG12
+  local_networks:
+    - pxe-rack1
+---
+schema: 'drydock/Network/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: oob
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  cidr: 172.16.100.1/24
+  ranges:
+    - type: static
+      start: 172.16.100.15
+      end: 172.16.100.254
+  dns:
+    domain: ilo.sitename.att.com
+    servers: 172.16.100.10
+---
+schema: 'drydock/Network/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: pxe
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  dhcp_relay:
+    self_ip: 172.16.0.4
+    upstream_target: 172.16.5.5
+  mtu: 1500
+  cidr: 172.16.0.0/24
+  ranges:
+    - type: dhcp
+      start: 172.16.0.5
+      end: 172.16.0.254
+  dns:
+    domain: admin.sitename.att.com
+    servers: 172.16.0.10
+---
+schema: 'drydock/Network/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: mgmt
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  vlan: '100'
+  mtu: 1500
+  cidr: 172.16.1.0/24
+  ranges:
+    - type: static
+      start: 172.16.1.15
+      end: 172.16.1.254
+  routes:
+    - subnet: 0.0.0.0/0
+      gateway: 172.16.1.1
+      metric: 10
+  dns:
+    domain: mgmt.sitename.example.com
+    servers: 172.16.1.9,172.16.1.10
+---
+schema: 'drydock/Network/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: private
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  vlan: '101'
+  mtu: 9000
+  cidr: 172.16.2.0/24
+  ranges:
+    - type: static
+      start: 172.16.2.15
+      end: 172.16.2.254
+  dns:
+    domain: priv.sitename.example.com
+    servers: 172.16.2.9,172.16.2.10
+---
+schema: 'drydock/Network/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: public
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  vlan: '102'
+  mtu: 1500
+  cidr: 172.16.3.0/24
+  ranges:
+    - type: static
+      start: 172.16.3.15
+      end: 172.16.3.254
+  routes:
+    - subnet: 0.0.0.0/0
+      gateway: 172.16.3.1
+      metric: 10
+  dns:
+    domain: sitename.example.com
+    servers: 8.8.8.8
+---
+schema: 'drydock/HostProfile/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: defaults
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  oob:
+    type: ipmi
+    network: oob
+    account: admin
+    credential: admin
+  storage:
+    physical_devices:
+      sda:
+        labels:
+          role: rootdisk
+        partitions:
+          - name: root
+            size: 20g
+            bootable: true
+            filesystem:
+              mountpoint: '/'
+              fstype: 'ext4'
+              mount_options: 'defaults'
+          - name: boot
+            size: 1g
+            bootable: false
+            filesystem:
+              mountpoint: '/boot'
+              fstype: 'ext4'
+              mount_options: 'defaults'
+      sdb:
+        volume_group: 'log_vg'
+    volume_groups:
+      log_vg:
+        logical_volumes:
+          - name: 'log_lv'
+            size: '500m'
+            filesystem:
+              mountpoint: '/var/log'
+              fstype: 'xfs'
+              mount_options: 'defaults'
+  platform:
+    image: 'xenial'
+    kernel: 'ga-16.04'
+    kernel_params:
+      quiet: true
+      console: ttyS2
+  metadata:
+    owner_data:
+      foo: bar
+---
+schema: 'drydock/HostProfile/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: 'k8-node'
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  host_profile: defaults
+  hardware_profile: HPGen9v3
+  primary_network: mgmt
+  interfaces:
+    pxe:
+      device_link: pxe
+      labels:
+        noconfig: true
+      slaves:
+        - prim_nic01
+      networks:
+        - pxe
+    bond0:
+      device_link: gp
+      slaves:
+        - prim_nic01
+        - prim_nic02
+      networks:
+        - mgmt
+        - private
+  metadata:
+    tags:
+      - 'test'
+---
+schema: 'drydock/BaremetalNode/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: controller01
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  host_profile: k8-node
+  interfaces:
+    bond0:
+      networks:
+        - '!private'
+  addressing:
+    - network: pxe
+      address: dhcp
+    - network: mgmt
+      address: 172.16.1.20
+    - network: public
+      address: 172.16.3.20
+    - network: oob
+      address: 172.16.100.20
+  metadata:
+    rack: rack1
+---
+schema: 'drydock/BaremetalNode/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: compute01
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  host_profile: k8-node
+  addressing:
+    - network: pxe
+      address: dhcp
+    - network: mgmt
+      address: 172.16.1.21
+    - network: private
+      address: 172.16.2.21
+    - network: oob
+      address: 172.16.100.21
+  metadata:
+    rack: rack2
+---
+schema: 'drydock/HardwareProfile/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: HPGen9v3
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  vendor: HP
+  generation: '8'
+  hw_version: '3'
+  bios_version: '2.2.3'
+  boot_mode: bios
+  bootstrap_protocol: pxe
+  pxe_interface: 0
+  device_aliases:
+    prim_nic01:
+      address: '0000:00:03.0'
+      dev_type: '82540EM Gigabit Ethernet Controller'
+      bus_type: 'pci'
+    prim_nic02:
+      address: '0000:00:04.0'
+      dev_type: '82540EM Gigabit Ethernet Controller'
+      bus_type: 'pci'
+    primary_boot:
+      address: '2:0.0.0'
+      dev_type: 'VBOX HARDDISK'
+      bus_type: 'scsi'
+---
+schema: 'drydock/BootAction/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: helloworld
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  assets:
+    - path: /var/tmp/hello.sh
+      type: file
+      permissions: '555'
+      data: |-
+        IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19
+        Jwo=
+      data_pipeline:
+        - base64_decode
+        - utf8_decode
+        - template
+    - path: /lib/systemd/system/hello.service
+      type: unit
+      permissions: '600'
+      data: |-
+        W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4
+        ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu
+        dGFyZ2V0Cg==
+      data_pipeline:
+        - base64_decode
+        - utf8_decode
+---
+schema: 'drydock/BootAction/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: hw_filtered
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  node_filter:
+    filter_set_type: 'union'
+    filter_set:
+      - filter_type: 'union'
+        node_names:
+          - 'compute01'
+  assets:
+    - path: /var/tmp/hello.sh
+      type: file
+      permissions: '555'
+      data: |-
+        IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19
+        Jwo=
+      data_pipeline:
+        - base64_decode
+        - utf8_decode
+        - template
+    - path: /lib/systemd/system/hello.service
+      type: unit
+      permissions: '600'
+      data: |-
+        W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4
+        ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu
+        dGFyZ2V0Cg==
+      data_pipeline:
+        - base64_decode
+        - utf8_decode
+...