From 4abd0a3184028a3412667f73d0e6064ae2a56d52 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Wed, 10 Apr 2019 16:34:42 +1000
Subject: [PATCH] yamlgroup: add regex match; exclude puppet4 for arm64 mirrors
Two related changes that need to go together because we test with the
production groups.yaml.
Confusingly, there are arm64 PC1 puppet repos, and it contains a bunch
of things that it turns out are the common java parts only. The
puppet-agent package is not available, and it doesn't seem like it
will be [1]. I think this means we can not run puppet4 on our arm64
xenial ci hosts.
The problem is the mirrors have been updated to puppet4 -- runs are
now breaking on the arm mirrors because they don't have puppet-agent
packages. It seems all we can really do at this point is contine to
run them on puppet3.
This is hard (impossible?) to express with a fnmatch in the existing
yamlgroups syntax. We could do something like list all the mirror
hosts and use anchors etc, but we have to keep that maintained. Add
an feature to the inventory plugin that if the list entry starts with
a ^ it is considered a full regex and passed to re.match. This
allows us to write more complex matchers where required -- in this
case the arm64 ci mirror hosts are excluded from the puppet4 group.
Testing is updated.
[1] https://groups.google.com/forum/#!msg/puppet-dev/iBMYJpvhaWM/WTGmJvXxAgAJ
Change-Id: I828e0c524f8d5ca866786978486bc04829464b47
---
inventory/groups.yaml | 2 +-
.../test-fixtures/results.yaml | 5 +++++
.../files/inventory_plugins/test_yamlgroup.py | 4 ++--
.../files/inventory_plugins/yamlgroup.py | 17 +++++++++++++++--
4 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/inventory/groups.yaml b/inventory/groups.yaml
index 58c3be7fec..0edfba0b88 100644
--- a/inventory/groups.yaml
+++ b/inventory/groups.yaml
@@ -175,7 +175,7 @@ groups:
- logstash-worker[0-9]*.open*.org
- logstash[0-9]*.open*.org
- mirror-update[0-9]*.open*.org
- - mirror[0-9]*.*.*.open*.org
+ - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.open.*\.org
- openstackid[0-9]*.openstack.org
- openstackid-dev[0-9]*.openstack.org
- paste[0-9]*.open*.org
diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
index 33add16ce5..23f8082c2c 100644
--- a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
+++ b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
@@ -39,6 +39,11 @@ results:
- puppet
- puppet4
+ mirror01.lon1.linaro-london.openstack.org:
+ - afs-client
+ - mirror
+ - puppet
+
mirror-update01.openstack.org:
- afsadmin
- puppet
diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py b/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py
index 7562ddb4fe..f6b18733fe 100644
--- a/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py
+++ b/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py
@@ -37,7 +37,7 @@ class TestInventory(testtools.TestCase):
results_yaml = os.path.join(FIXTURE_DIR, 'results.yaml')
with open(results_yaml) as f:
- results = yaml.load(f)
+ results = yaml.load(f, Loader=yaml.FullLoader)
results = results['results']
# Build the inventory list. This is a list of Host objects
@@ -65,7 +65,7 @@ class TestInventory(testtools.TestCase):
# real-life, which gets the groups into the config object
path = os.path.join(FIXTURE_DIR, 'groups.yaml')
with open(path) as f:
- config_groups = yaml.load(f)
+ config_groups = yaml.load(f, Loader=yaml.FullLoader)
config_groups = config_groups['groups']
im = InventoryModule()
im._read_config_data = mock.MagicMock()
diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py b/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py
index 4fb41cae83..1b47315232 100644
--- a/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py
+++ b/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py
@@ -3,6 +3,7 @@
import fnmatch
import os
+import re
from ansible.parsing.yaml.objects import AnsibleMapping
from ansible.plugins.inventory import BaseFileInventoryPlugin
@@ -28,7 +29,10 @@ DOCUMENTATION = '''
- section: inventory_plugin_yaml
key: yaml_valid_extensions
groups:
- description: dict with group name as key and list of fnmatch patterns
+ description: |
+ dict with group name as key. If the list item starts with a
+ ^ it will be considered a regex pattern (i.e. passed to
+ re.match), otherwise it is considered a fnmatch pattern.
type: dict
default: {}
'''
@@ -38,6 +42,7 @@ groups:
amazing:
- fullhost.example.com
- amazing*
+ - ^regex.*pattern
'''
@@ -75,8 +80,16 @@ class InventoryModule(BaseFileInventoryPlugin):
# failing.
if isinstance(candidate, AnsibleMapping):
candidate = list(candidate.keys())[0]
+
+ # Starts with ^ means it is already a regex.
+ # Otherwise it's a fnmatch compatible string; use it's
+ # helper to turn that into a regex so we have a common
+ # match below.
+ if not candidate.startswith('^'):
+ candidate = fnmatch.translate(candidate)
+
for existing in self.inventory.hosts.values():
- if fnmatch.fnmatch(existing.get_name(), candidate):
+ if re.match(candidate, existing.get_name()):
found_groups.setdefault(group, [])
found_groups[group].append(existing)